Adds candidate formatter for replacing legacy summary results.

Also cleans up some usages of strings where symbolic names
were safer and made more sense.

Try a test run with something like this to check out the new
basic results formatter (not used by default):

time test/dotest.py --executable `pwd`/build/Debug/lldb --results-formatter lldbsuite.test.basic_results_formatter.BasicResultsFormatter --results-file stdout

This will yield something like:

Testing: 1 test suites, 8 threads
1 out of 1 test suites processed - TestHelp.py
Test Results
Total Test Methods Run (excluding reruns): 13
Test Method rerun count: 0

===================
Test Result Summary
===================
Success:              13
Expected Failure:      0
Failure:               0
Error:                 0
Unexpected Success:    0
Skip:                  0

Whereas something with a bit of error will look more like this:

42 out of 42 test suites processed - TestSymbolTable.py
Test Results
Total Test Methods Run (excluding reruns): 166
Test Method rerun count: 0

===================
Test Result Summary
===================
Success:              93
Expected Failure:     10
Failure:               2
Error:                 2
Unexpected Success:    0
Skip:                 59

Details:
FAIL:
TestModulesInlineFunctions.ModulesInlineFunctionsTestCase.test_expr_dsym
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py)
FAIL:
TestModulesInlineFunctions.ModulesInlineFunctionsTestCase.test_expr_dwarf
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py)
ERROR: TestObjCCheckers.ObjCCheckerTestCase.test_objc_checker_dsym
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py)
ERROR: TestObjCCheckers.ObjCCheckerTestCase.test_objc_checker_dwarf
(/Users/tfiala/work/lldb-tot/git-svn/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py)

The Details header only prints if there are any issues to report.  The
Details section has tags that should get picked up using the normal
issue text scrapers (e.g. buildbot).

Test numbers reported are strictly test method runs.

The rerun bit at the top is in support of the multi-pass test
runner code (to run the low-load, single worker test pass for
tests that failed the first run), which I'll be able to put up
for review after this.

ResultsFormatters now have the ability to indicate they replace
the legacy summary, as this one does.

Once we come to agreement on the exact format, I will switch
us over to using this by default.

llvm-svn: 254530

1 files changed, 74 insertions, 23 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/test_results.py b/lldb/packages/Python/lldbsuite/test/test_results.py
index 6f9cefe31d6..12fb2e51d77 100644
--- a/lldb/packages/Python/lldbsuite/test/test_results.py
+++ b/lldb/packages/Python/lldbsuite/test/test_results.py
@@ -29,11 +29,20 @@ from six.moves import cPickle
 
 # LLDB modules
 
+
 class EventBuilder(object):
     """Helper class to build test result event dictionaries."""
 
     BASE_DICTIONARY = None
 
+    # Test Status Tags
+    STATUS_SUCCESS = "success"
+    STATUS_FAILURE = "failure"
+    STATUS_EXPECTED_FAILURE = "expected_failure"
+    STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
+    STATUS_SKIP = "skip"
+    STATUS_ERROR = "error"
+
     @staticmethod
     def _get_test_name_info(test):
         """Returns (test-class-name, test-method-name) from a test case instance.
@@ -182,7 +191,8 @@ class EventBuilder(object):
 
         @return the event dictionary
         """
-        return EventBuilder._event_dictionary_test_result(test, "success")
+        return EventBuilder._event_dictionary_test_result(
+            test, EventBuilder.STATUS_SUCCESS)
 
     @staticmethod
     def event_for_unexpected_success(test, bugnumber):
@@ -199,7 +209,7 @@ class EventBuilder(object):
 
         """
         event = EventBuilder._event_dictionary_test_result(
-            test, "unexpected_success")
+            test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
         if bugnumber:
             event["bugnumber"] = str(bugnumber)
         return event
@@ -216,7 +226,7 @@ class EventBuilder(object):
         @return the event dictionary
         """
         return EventBuilder._event_dictionary_issue(
-            test, "failure", error_tuple)
+            test, EventBuilder.STATUS_FAILURE, error_tuple)
 
     @staticmethod
     def event_for_expected_failure(test, error_tuple, bugnumber):
@@ -234,7 +244,7 @@ class EventBuilder(object):
 
         """
         event = EventBuilder._event_dictionary_issue(
-            test, "expected_failure", error_tuple)
+            test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
         if bugnumber:
             event["bugnumber"] = str(bugnumber)
         return event
@@ -249,7 +259,8 @@ class EventBuilder(object):
 
         @return the event dictionary
         """
-        event = EventBuilder._event_dictionary_test_result(test, "skip")
+        event = EventBuilder._event_dictionary_test_result(
+            test, EventBuilder.STATUS_SKIP)
         event["skip_reason"] = reason
         return event
 
@@ -264,7 +275,8 @@ class EventBuilder(object):
 
         @return the event dictionary
         """
-        return EventBuilder._event_dictionary_issue(test, "error", error_tuple)
+        return EventBuilder._event_dictionary_issue(
+            test, EventBuilder.STATUS_ERROR, error_tuple)
 
     @staticmethod
     def event_for_cleanup_error(test, error_tuple):
@@ -279,7 +291,7 @@ class EventBuilder(object):
         @return the event dictionary
         """
         event = EventBuilder._event_dictionary_issue(
-            test, "error", error_tuple)
+            test, EventBuilder.STATUS_ERROR, error_tuple)
         event["issue_phase"] = "cleanup"
         return event
 
@@ -376,7 +388,6 @@ class ResultsFormatter(object):
     expectations about when the call should be chained.
 
     """
-
     @classmethod
     def arg_parser(cls):
         """@return arg parser used to parse formatter-specific options."""
@@ -396,6 +407,16 @@ class ResultsFormatter(object):
         self.start_time_by_test = {}
         self.terminate_called = False
 
+        # Store counts of test_result events by status.
+        self.result_status_counts = {
+            EventBuilder.STATUS_SUCCESS: 0,
+            EventBuilder.STATUS_EXPECTED_FAILURE: 0,
+            EventBuilder.STATUS_SKIP: 0,
+            EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0,
+            EventBuilder.STATUS_FAILURE: 0,
+            EventBuilder.STATUS_ERROR: 0
+        }
+
         # Lock that we use while mutating inner state, like the
         # total test count and the elements.  We minimize how
         # long we hold the lock just to keep inner state safe, not
@@ -417,14 +438,19 @@ class ResultsFormatter(object):
         # atexit() cleanup can call the "terminate if it hasn't been
         # called yet".
         if test_event is not None:
-            if test_event.get("event", "") == "terminate":
+            event_type = test_event.get("event", "")
+            if event_type == "terminate":
                 self.terminate_called = True
+            elif event_type == "test_result":
+                # Keep track of event counts per test result status type
+                status = test_event["status"]
+                self.result_status_counts[status] += 1
 
     def track_start_time(self, test_class, test_name, start_time):
-        """Tracks the start time of a test so elapsed time can be computed.
+        """tracks the start time of a test so elapsed time can be computed.
 
-        This alleviates the need for test results to be processed serially
-        by test.  It will save the start time for the test so that
+        this alleviates the need for test results to be processed serially
+        by test.  it will save the start time for the test so that
         elapsed_time_for_test() can compute the elapsed time properly.
         """
         if test_class is None or test_name is None:
@@ -435,9 +461,9 @@ class ResultsFormatter(object):
             self.start_time_by_test[test_key] = start_time
 
     def elapsed_time_for_test(self, test_class, test_name, end_time):
-        """Returns the elapsed time for a test.
+        """returns the elapsed time for a test.
 
-        This function can only be called once per test and requires that
+        this function can only be called once per test and requires that
         the track_start_time() method be called sometime prior to calling
         this method.
         """
@@ -454,16 +480,38 @@ class ResultsFormatter(object):
         return end_time - start_time
 
     def is_using_terminal(self):
-        """Returns True if this results formatter is using the terminal and
+        """returns true if this results formatter is using the terminal and
         output should be avoided."""
         return self.using_terminal
 
     def send_terminate_as_needed(self):
-        """Sends the terminate event if it hasn't been received yet."""
+        """sends the terminate event if it hasn't been received yet."""
         if not self.terminate_called:
             terminate_event = EventBuilder.bare_event("terminate")
             self.handle_event(terminate_event)
 
+    # Derived classes may require self access
+    # pylint: disable=no-self-use
+    def replaces_summary(self):
+        """Returns whether the results formatter includes a summary
+        suitable to replace the old lldb test run results.
+
+        @return True if the lldb test runner can skip its summary
+        generation when using this results formatter; False otherwise.
+        """
+        return False
+
+    def counts_by_test_result_status(self, status):
+        """Returns number of test method results for the given status.
+
+        @status_result a test result status (e.g. success, fail, skip)
+        as defined by the EventBuilder.STATUS_* class members.
+
+        @return an integer returning the number of test methods matching
+        the given test result status.
+        """
+        return self.result_status_counts[status]
+
 
 class XunitFormatter(ResultsFormatter):
     """Provides xUnit-style formatted output.
@@ -527,7 +575,8 @@ class XunitFormatter(ResultsFormatter):
             unicode_content = str_or_unicode.decode('utf-8')
         else:
             unicode_content = str_or_unicode
-        return self.invalid_xml_re.sub(six.u('?'), unicode_content).encode('utf-8')
+        return self.invalid_xml_re.sub(
+            six.u('?'), unicode_content).encode('utf-8')
 
     @classmethod
     def arg_parser(cls):
@@ -622,12 +671,14 @@ class XunitFormatter(ResultsFormatter):
             }
 
         self.status_handlers = {
-            "success": self._handle_success,
-            "failure": self._handle_failure,
-            "error": self._handle_error,
-            "skip": self._handle_skip,
-            "expected_failure": self._handle_expected_failure,
-            "unexpected_success": self._handle_unexpected_success
+            EventBuilder.STATUS_SUCCESS: self._handle_success,
+            EventBuilder.STATUS_FAILURE: self._handle_failure,
+            EventBuilder.STATUS_ERROR: self._handle_error,
+            EventBuilder.STATUS_SKIP: self._handle_skip,
+            EventBuilder.STATUS_EXPECTED_FAILURE:
+                self._handle_expected_failure,
+            EventBuilder.STATUS_UNEXPECTED_SUCCESS:
+                self._handle_unexpected_success
             }
 
     def handle_event(self, test_event):