surface build error content through test event system

Summary:
print build errors nicely in test output

This test infrastructure change adds a new Python exception
for test subject builds that fail.  The output of the build
command is captured and propagated to both the textual test
output display code and to the test event system.

The ResultsFormatter objects have been modified to do something
more useful with this information.  The xUnit formatter
now replaces the non-informative Python build error stacktrace
with the build error content.  The curses ResultsFormatter
prints a 'B' for build errors rather than 'E'.

The xUnit output, in particular, makes it much easier for
developers to track down test subject build errors that cause
test failures when reports come in from CI.

Reviewers: granata.enrico

Subscribers: llvm-commits

Differential Revision: http://reviews.llvm.org/D20252

llvm-svn: 269525

2 files changed, 37 insertions, 5 deletions

diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py b/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
index 07b33860974..4e89fa9bf01 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
@@ -59,7 +59,7 @@ class Curses(results_formatter.ResultsFormatter):
         # if tee_results_formatter:
         #     self.formatters.append(tee_results_formatter)
 
-    def status_to_short_str(self, status):
+    def status_to_short_str(self, status, test_event):
         if status == EventBuilder.STATUS_SUCCESS:
             return '.'
         elif status == EventBuilder.STATUS_FAILURE:
@@ -71,7 +71,11 @@ class Curses(results_formatter.ResultsFormatter):
         elif status == EventBuilder.STATUS_SKIP:
             return 'S'
         elif status == EventBuilder.STATUS_ERROR:
-            return 'E'
+            if test_event.get("issue_phase", None) == "build":
+                # Build failure
+                return 'B'
+            else:
+                return 'E'
         elif status == EventBuilder.STATUS_TIMEOUT:
             return 'T'
         elif status == EventBuilder.STATUS_EXPECTED_TIMEOUT:
@@ -123,7 +127,7 @@ class Curses(results_formatter.ResultsFormatter):
             if status in self.hide_status_list:
                 continue
             name = test_result['test_class'] + '.' + test_result['test_name']
-            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status), test_result['elapsed_time'], name))
+            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_result), test_result['elapsed_time'], name))
         if update:
             self.main_window.refresh()
 
@@ -162,7 +166,7 @@ class Curses(results_formatter.ResultsFormatter):
                         name = test_event['test_class'] + '.' + test_event['test_name']
                         elapsed_time = test_event['event_time'] - self.job_tests[worker_index]['event_time']
                         if not status in self.hide_status_list:
-                            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status), elapsed_time, name))
+                            self.results_panel.append_line('%s (%6.2f sec) %s' % (self.status_to_short_str(status, test_event), elapsed_time, name))
                         self.main_window.refresh()
                         # Append the result pairs
                         test_event['elapsed_time'] = elapsed_time
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py b/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
index 94f843a4ff8..b3682f8fb10 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
@@ -21,6 +21,7 @@ import six
 
 # Local modules
 from ..event_builder import EventBuilder
+from ..build_exception import BuildError
 from .results_formatter import ResultsFormatter
 
 
@@ -246,7 +247,28 @@ class XunitFormatter(ResultsFormatter):
         with self.lock:
             self.elements["failures"].append(result)
 
-    def _handle_error(self, test_event):
+    def _handle_error_build(self, test_event):
+        """Handles a test error.
+        @param test_event the test event to handle.
+        """
+        message = self._replace_invalid_xml(test_event["issue_message"])
+        build_issue_description = self._replace_invalid_xml(
+            BuildError.format_build_error(
+                test_event.get("build_command", "<None>"),
+                test_event.get("build_error", "<None>")))
+
+        result = self._common_add_testcase_entry(
+            test_event,
+            inner_content=(
+                '<error type={} message={}><![CDATA[{}]]></error>'.format(
+                    XunitFormatter._quote_attribute(test_event["issue_class"]),
+                    XunitFormatter._quote_attribute(message),
+                    build_issue_description)
+            ))
+        with self.lock:
+            self.elements["errors"].append(result)
+
+    def _handle_error_standard(self, test_event):
         """Handles a test error.
         @param test_event the test event to handle.
         """
@@ -265,6 +287,12 @@ class XunitFormatter(ResultsFormatter):
         with self.lock:
             self.elements["errors"].append(result)
 
+    def _handle_error(self, test_event):
+        if test_event.get("issue_phase", None) == "build":
+            self._handle_error_build(test_event)
+        else:
+            self._handle_error_standard(test_event)
+
     def _handle_exceptional_exit(self, test_event):
         """Handles an exceptional exit.
         @param test_event the test method or job result event to handle.