surface build error content through test event system

Summary:
print build errors nicely in test output

This test infrastructure change adds a new Python exception
for test subject builds that fail.  The output of the build
command is captured and propagated to both the textual test
output display code and to the test event system.

The ResultsFormatter objects have been modified to do something
more useful with this information.  The xUnit formatter
now replaces the non-informative Python build error stacktrace
with the build error content.  The curses ResultsFormatter
prints a 'B' for build errors rather than 'E'.

The xUnit output, in particular, makes it much easier for
developers to track down test subject build errors that cause
test failures when reports come in from CI.

Reviewers: granata.enrico

Subscribers: llvm-commits

Differential Revision: http://reviews.llvm.org/D20252

llvm-svn: 269525

4 files changed, 66 insertions, 13 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 492e136fe39..14b9876e407 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -419,7 +419,14 @@ def system(commands, **kwargs):
             cmd = kwargs.get("args")
             if cmd is None:
                 cmd = shellCommand
-            raise CalledProcessError(retcode, cmd)
+            cpe = CalledProcessError(retcode, cmd)
+            # Ensure caller can access the stdout/stderr.
+            cpe.lldb_extensions = {
+                "stdout_content": this_output,
+                "stderr_content": this_error,
+                "command": shellCommand
+            }
+            raise cpe
         output = output + this_output
         error = error + this_error
     return (output, error)
diff --git a/lldb/packages/Python/lldbsuite/test/plugins/builder_base.py b/lldb/packages/Python/lldbsuite/test/plugins/builder_base.py
index 0cff14c2269..6b8373b453c 100644
--- a/lldb/packages/Python/lldbsuite/test/plugins/builder_base.py
+++ b/lldb/packages/Python/lldbsuite/test/plugins/builder_base.py
@@ -12,10 +12,16 @@ Same idea holds for LLDB_ARCH environment variable, which maps to the ARCH make
 variable.
 """
 
-import os, sys
+# System imports
+import os
 import platform
+import subprocess
+import sys
+
+# Our imports
 import lldbsuite.test.lldbtest as lldbtest
 import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test_event import build_exception
 
 def getArchitecture():
     """Returns the architecture in effect the test suite is running with."""
@@ -93,6 +99,16 @@ def getCmdLine(d):
     return cmdline
 
 
+def runBuildCommands(commands, sender):
+    try:
+        lldbtest.system(commands, sender=sender)
+    except subprocess.CalledProcessError as called_process_error:
+        # Convert to a build-specific error.
+        # We don't do that in lldbtest.system() since that
+        # is more general purpose.
+        raise build_exception.BuildError(called_process_error)
+
+
 def buildDefault(sender=None, architecture=None, compiler=None, dictionary=None, clean=True):
     """Build the binaries the default way."""
     commands = []
@@ -100,7 +116,7 @@ def buildDefault(sender=None, architecture=None, compiler=None, dictionary=None,
         commands.append([getMake(), "clean", getCmdLine(dictionary)])
     commands.append([getMake(), getArchSpec(architecture), getCCSpec(compiler), getCmdLine(dictionary)])
 
-    lldbtest.system(commands, sender=sender)
+    runBuildCommands(commands, sender=sender)
 
     # True signifies that we can handle building default.
     return True
@@ -112,7 +128,7 @@ def buildDwarf(sender=None, architecture=None, compiler=None, dictionary=None, c
         commands.append([getMake(), "clean", getCmdLine(dictionary)])
     commands.append([getMake(), "MAKE_DSYM=NO", getArchSpec(architecture), getCCSpec(compiler), getCmdLine(dictionary)])
 
-    lldbtest.system(commands, sender=sender)
+    runBuildCommands(commands, sender=sender)
     # True signifies that we can handle building dwarf.
     return True
 
@@ -123,7 +139,7 @@ def buildDwo(sender=None, architecture=None, compiler=None, dictionary=None, cle
         commands.append([getMake(), "clean", getCmdLine(dictionary)])
     commands.append([getMake(), "MAKE_DSYM=NO", "MAKE_DWO=YES", getArchSpec(architecture), getCCSpec(compiler), getCmdLine(dictionary)])
 
-    lldbtest.system(commands, sender=sender)
+    runBuildCommands(commands, sender=sender)
     # True signifies that we can handle building dwo.
     return True
 
@@ -135,6 +151,6 @@ def cleanup(sender=None, dictionary=None):
     if os.path.isfile("Makefile"):
         commands.append([getMake(), "clean", getCmdLine(dictionary)])
 
-    lldbtest.system(commands, sender=sender)
+    runBuildCommands(commands, sender=sender)
     # True signifies that we can handle cleanup.
     return True
diff --git a/lldb/packages/Python/lldbsuite/test/plugins/builder_darwin.py b/lldb/packages/Python/lldbsuite/test/plugins/builder_darwin.py
index dd07206e323..8a907ccec2d 100644
--- a/lldb/packages/Python/lldbsuite/test/plugins/builder_darwin.py
+++ b/lldb/packages/Python/lldbsuite/test/plugins/builder_darwin.py
@@ -5,8 +5,6 @@ import lldbsuite.test.lldbtest as lldbtest
 
 from builder_base import *
 
-#print("Hello, darwin plugin!")
-
 def buildDsym(sender=None, architecture=None, compiler=None, dictionary=None, clean=True):
     """Build the binaries with dsym debug info."""
     commands = []
@@ -15,7 +13,7 @@ def buildDsym(sender=None, architecture=None, compiler=None, dictionary=None, cl
         commands.append(["make", "clean", getCmdLine(dictionary)])
     commands.append(["make", "MAKE_DSYM=YES", getArchSpec(architecture), getCCSpec(compiler), getCmdLine(dictionary)])
 
-    lldbtest.system(commands, sender=sender)
+    runBuildCommands(commands, sender=sender)
 
     # True signifies that we can handle building dsym.
     return True
diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py
index 949ebc84362..3b4b0f4486b 100644
--- a/lldb/packages/Python/lldbsuite/test/test_result.py
+++ b/lldb/packages/Python/lldbsuite/test/test_result.py
@@ -13,6 +13,7 @@ from __future__ import print_function
 
 # System modules
 import inspect
+import os
 
 # Third-party modules
 import unittest2
@@ -20,7 +21,7 @@ import unittest2
 # LLDB Modules
 from . import configuration
 from lldbsuite.test_event.event_builder import EventBuilder
-
+from lldbsuite.test_event import build_exception
 
 class LLDBTestResult(unittest2.TextTestResult):
     """
@@ -139,17 +140,48 @@ class LLDBTestResult(unittest2.TextTestResult):
             self.results_formatter.handle_event(
                 EventBuilder.event_for_success(test))
 
+    def _isBuildError(self, err_tuple):
+        exception = err_tuple[1]
+        return isinstance(exception, build_exception.BuildError)
+
+    def _getTestPath(self, test):
+        if test is None:
+            return ""
+        elif hasattr(test, "test_filename"):
+            return test.test_filename
+        else:
+            return inspect.getsourcefile(test.__class__)
+
+
+    def _saveBuildErrorTuple(self, test, err):
+        # Adjust the error description so it prints the build command and build error
+        # rather than an uninformative Python backtrace.
+        build_error = err[1]
+        error_description = "{}\nTest Directory:\n{}".format(
+            str(build_error),
+            os.path.dirname(self._getTestPath(test)))
+        self.errors.append((test, error_description))
+        self._mirrorOutput = True
+
     def addError(self, test, err):
         configuration.sdir_has_content = True
-        super(LLDBTestResult, self).addError(test, err)
+        if self._isBuildError(err):
+            self._saveBuildErrorTuple(test, err)
+        else:
+            super(LLDBTestResult, self).addError(test, err)
+
         method = getattr(test, "markError", None)
         if method:
             method()
         if configuration.parsable:
             self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
         if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_error(test, err))
+            # Handle build errors as a separate event type
+            if self._isBuildError(err):
+                error_event = EventBuilder.event_for_build_error(test, err)
+            else:
+                error_event = EventBuilder.event_for_error(test, err)
+            self.results_formatter.handle_event(error_event)
 
     def addCleanupError(self, test, err):
         configuration.sdir_has_content = True