test infra: catch bad decorators and import-time errors

Summary:
This change enhances the LLDB test infrastructure to convert
load-time exceptions in a given Python test module into errors.
Before this change, specifying a non-existent test decorator,
or otherwise having some load-time error in a python test module,
would not get flagged as an error.

With this change, typos and other load-time errors in a python
test file get converted to errors and reported by the
test runner.

This change also includes test infrastructure tests that include
covering the new work here.  I'm going to wait until we have
these infrastructure tests runnable on the main platforms before
I try to work that into all the normal testing workflows.

The test infrastructure tests can be run by using the standard python module testing practice of doing the following:

cd packages/Python/lldbsuite/test_event
python -m unittest discover -s test/src -p 'Test*.py'

Those tests run the dotest inferior with a known broken test and verify that the errors are caught.  These tests did not pass until I modified dotest.py to capture them properly.

@zturner, if you have the chance, if you could try those steps above (the python -m unittest ... line) on Windows, that would be great if we can address any python2/3/Windows bits there.  I don't think there's anything fancy, but I didn't want to hook it into test flow until I know it works there.

I'll be slowly adding more tests that cover some of the other breakage I've occasionally seen that didn't get collected as part of the summarization.  This is the biggest one I'm aware of.

Reviewers: zturner, labath

Subscribers: zturner, lldb-commits

Differential Revision: http://reviews.llvm.org/D20193

llvm-svn: 269489

5 files changed, 34 insertions, 13 deletions

diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py b/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py
index 5a07af6c41a..556370ebb9d 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py
@@ -84,6 +84,7 @@ def create_results_formatter(config):
     results_file_object = None
     cleanup_func = None
 
+    file_is_stream = False
     if config.filename:
         # Open the results file for writing.
         if config.filename == 'stdout':
@@ -102,6 +103,7 @@ def create_results_formatter(config):
         results_file_object, cleanup_func = create_socket(config.port)
         default_formatter_name = (
             "lldbsuite.test_event.formatter.pickled.RawPickledFormatter")
+        file_is_stream = True
 
     # If we have a results formatter name specified and we didn't specify
     # a results file, we should use stdout.
@@ -137,7 +139,10 @@ def create_results_formatter(config):
             command_line_options)
 
         # Create the TestResultsFormatter given the processed options.
-        results_formatter_object = cls(results_file_object, formatter_options)
+        results_formatter_object = cls(
+            results_file_object,
+            formatter_options,
+            file_is_stream)
 
         def shutdown_formatter():
             """Shuts down the formatter when it is no longer needed."""
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py b/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
index 85782599a40..07b33860974 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/curses.py
@@ -27,9 +27,9 @@ from ..event_builder import EventBuilder
 class Curses(results_formatter.ResultsFormatter):
     """Receives live results from tests that are running and reports them to the terminal in a curses GUI"""
 
-    def __init__(self, out_file, options):
+    def __init__(self, out_file, options, file_is_stream):
         # Initialize the parent
-        super(Curses, self).__init__(out_file, options)
+        super(Curses, self).__init__(out_file, options, file_is_stream)
         self.using_terminal = True
         self.have_curses = True
         self.initialize_event = None
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/pickled.py b/lldb/packages/Python/lldbsuite/test_event/formatter/pickled.py
index 9607e91fd55..6d800f6c8ba 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/pickled.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/pickled.py
@@ -30,9 +30,27 @@ class RawPickledFormatter(ResultsFormatter):
         parser = super(RawPickledFormatter, cls).arg_parser()
         return parser
 
-    def __init__(self, out_file, options):
-        super(RawPickledFormatter, self).__init__(out_file, options)
+    class StreamSerializer(object):
+        @staticmethod
+        def serialize(test_event, out_file):
+            # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes}
+            import struct
+            msg = cPickle.dumps(test_event)
+            packet = struct.pack("!I%ds" % len(msg), len(msg), msg)
+            out_file.send(packet)
+
+    class BlockSerializer(object):
+        @staticmethod
+        def serialize(test_event, out_file):
+            cPickle.dump(test_event, out_file)
+
+    def __init__(self, out_file, options, file_is_stream):
+        super(RawPickledFormatter, self).__init__(out_file, options, file_is_stream)
         self.pid = os.getpid()
+        if file_is_stream:
+            self.serializer = self.StreamSerializer()
+        else:
+            self.serializer = self.BlockSerializer()
 
     def handle_event(self, test_event):
         super(RawPickledFormatter, self).handle_event(test_event)
@@ -50,8 +68,5 @@ class RawPickledFormatter(ResultsFormatter):
         # Tack on the pid.
         test_event["pid"] = self.pid
 
-        # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes}
-        import struct
-        msg = cPickle.dumps(test_event)
-        packet = struct.pack("!I%ds" % len(msg), len(msg), msg)
-        self.out_file.send(packet)
+        # Serialize the test event.
+        self.serializer.serialize(test_event, self.out_file)
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py b/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py
index 0fc5649bad1..3bf389b9726 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py
@@ -111,7 +111,7 @@ class ResultsFormatter(object):
                   'the summary output.'))
         return parser
 
-    def __init__(self, out_file, options):
+    def __init__(self, out_file, options, file_is_stream):
         super(ResultsFormatter, self).__init__()
         self.out_file = out_file
         self.options = options
@@ -120,6 +120,7 @@ class ResultsFormatter(object):
             raise Exception("ResultsFormatter created with no file object")
         self.start_time_by_test = {}
         self.terminate_called = False
+        self.file_is_stream = file_is_stream
 
         # Store counts of test_result events by status.
         self.result_status_counts = {
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py b/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
index 02a49ed0834..94f843a4ff8 100644
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
+++ b/lldb/packages/Python/lldbsuite/test_event/formatter/xunit.py
@@ -153,14 +153,14 @@ class XunitFormatter(ResultsFormatter):
                 regex_list.append(re.compile(pattern))
         return regex_list
 
-    def __init__(self, out_file, options):
+    def __init__(self, out_file, options, file_is_stream):
         """Initializes the XunitFormatter instance.
         @param out_file file-like object where formatted output is written.
         @param options specifies a dictionary of options for the
         formatter.
         """
         # Initialize the parent
-        super(XunitFormatter, self).__init__(out_file, options)
+        super(XunitFormatter, self).__init__(out_file, options, file_is_stream)
         self.text_encoding = "UTF-8"
         self.invalid_xml_re = XunitFormatter._build_illegal_xml_regex()
         self.total_test_count = 0