test infra: adds book-keeping for rerunnable tests

Also adds full path info for exceptional exits and timeouts when
no test method is currently running.

Adds --rerun-all-issues command line arg.  If specified, all
test issues are eligible for rerun.  If not specified, only tests
marked flakey are eligible for rerun.

The actual rerunning will occur in an upcoming change.  This
change just handles tha accounting of what should be rerun.

llvm-svn: 255438

7 files changed, 165 insertions, 45 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py b/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
index aded67532a3..b2b227ba370 100644
--- a/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
+++ b/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
@@ -258,37 +258,34 @@ class BasicResultsFormatter(result_formatter.ResultsFormatter):
         if print_matching_tests:
             # Sort by test name
             for (_, event) in result_events_by_status[result_status_id]:
-                extra_info = ""
+                # Convert full test path into test-root-relative.
+                test_relative_path = os.path.relpath(
+                    os.path.realpath(event["test_filename"]),
+                    lldbsuite.lldb_test_root)
+
+                # Create extra info component (used for exceptional exit info)
                 if result_status_id == EventBuilder.STATUS_EXCEPTIONAL_EXIT:
-                    extra_info = "{} ({}) ".format(
+                    extra_info = "[EXCEPTIONAL EXIT {} ({})] ".format(
                         event["exception_code"],
                         event["exception_description"])
-
-                if event["event"] == EventBuilder.TYPE_JOB_RESULT:
-                    # Jobs status that couldn't be mapped to a test method
-                    # doesn't have as much detail.
-                    self.out_file.write(
-                        "{}: {}{} (no test method running)\n".format(
-                            detail_label,
-                            extra_info,
-                            event["test_filename"]))
                 else:
-                    # Figure out the identity we will use for this test.
-                    if configuration.verbose and ("test_class" in event):
-                        test_id = "{}.{}".format(
-                            event["test_class"], event["test_name"])
-                    else:
-                        test_id = event["test_name"]
-
-                    # Test-method events have richer detail, use that here.
-                    test_relative_path = os.path.relpath(
-                        os.path.realpath(event["test_filename"]),
-                        lldbsuite.lldb_test_root)
-                    self.out_file.write("{}: {}{} ({})\n".format(
-                        detail_label,
-                        extra_info,
-                        test_id,
-                        test_relative_path))
+                    extra_info = ""
+
+                # Figure out the identity we will use for this test.
+                if configuration.verbose and ("test_class" in event):
+                    test_id = "{}.{}".format(
+                        event["test_class"], event["test_name"])
+                elif "test_name" in event:
+                    test_id = event["test_name"]
+                else:
+                    test_id = "<no_running_test_method>"
+
+                # Display the info.
+                self.out_file.write("{}: {}{} ({})\n".format(
+                    detail_label,
+                    extra_info,
+                    test_id,
+                    test_relative_path))
 
     def _finish_output_no_lock(self):
         """Writes the test result report to the output file."""
diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index 7fd68b6dd12..d1aaef5cb7d 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -143,6 +143,9 @@ results_formatter_object = None
 results_formatter_options = None
 test_result = None
 
+# Test rerun configuration vars
+rerun_all_issues = False
+
 # The names of all tests. Used to assert we don't have two tests with the same base name.
 all_tests = set()
 
diff --git a/lldb/packages/Python/lldbsuite/test/dosep.py b/lldb/packages/Python/lldbsuite/test/dosep.py
index 31cc4b749e7..b3c22519b58 100644
--- a/lldb/packages/Python/lldbsuite/test/dosep.py
+++ b/lldb/packages/Python/lldbsuite/test/dosep.py
@@ -315,7 +315,8 @@ def send_events_to_collector(events, command):
         formatter_spec.cleanup_func()
 
 
-def send_inferior_post_run_events(command, worker_index, process_driver):
+def send_inferior_post_run_events(
+        command, worker_index, process_driver, test_filename):
     """Sends any test events that should be generated after the inferior runs.
 
     These events would include timeouts and exceptional (i.e. signal-returning)
@@ -326,6 +327,8 @@ def send_inferior_post_run_events(command, worker_index, process_driver):
     this process
     @param process_driver the ProcessDriver-derived instance that was used
     to run the inferior process.
+    @param test_filename the full path to the Python test file that is being
+    run.
     """
     if process_driver is None:
         raise Exception("process_driver must not be None")
@@ -342,7 +345,6 @@ def send_inferior_post_run_events(command, worker_index, process_driver):
     # Handle signal/exceptional exits.
     if process_driver.is_exceptional_exit():
         (code, desc) = process_driver.exceptional_exit_details()
-        test_filename = process_driver.results[0]
         post_events.append(
             EventBuilder.event_for_job_exceptional_exit(
                 process_driver.pid,
@@ -354,7 +356,6 @@ def send_inferior_post_run_events(command, worker_index, process_driver):
 
     # Handle timeouts.
     if process_driver.is_timeout():
-        test_filename = process_driver.results[0]
         post_events.append(EventBuilder.event_for_job_timeout(
             process_driver.pid,
             worker_index,
@@ -365,7 +366,8 @@ def send_inferior_post_run_events(command, worker_index, process_driver):
         send_events_to_collector(post_events, command)
 
 
-def call_with_timeout(command, timeout, name, inferior_pid_events):
+def call_with_timeout(
+        command, timeout, name, inferior_pid_events, test_filename):
     # Add our worker index (if we have one) to all test events
     # from this inferior.
     worker_index = None
@@ -405,8 +407,8 @@ def call_with_timeout(command, timeout, name, inferior_pid_events):
     send_inferior_post_run_events(
         command,
         worker_index,
-        process_driver)
-
+        process_driver,
+        test_filename)
 
     return process_driver.results
 
@@ -426,8 +428,9 @@ def process_dir(root, files, dotest_argv, inferior_pid_events):
         timeout = (os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or
                    getDefaultTimeout(dotest_options.lldb_platform_name))
 
+        test_filename = os.path.join(root, name)
         results.append(call_with_timeout(
-            command, timeout, name, inferior_pid_events))
+            command, timeout, name, inferior_pid_events, test_filename))
 
     # result = (name, status, passes, failures, unexpected_successes)
     timed_out = [name for name, status, _, _, _ in results
@@ -436,13 +439,15 @@ def process_dir(root, files, dotest_argv, inferior_pid_events):
               if status == ePassed]
     failed = [name for name, status, _, _, _ in results
               if status != ePassed]
-    unexpected_passes = [name for name, _, _, _, unexpected_successes in results
-                         if unexpected_successes > 0]
+    unexpected_passes = [
+        name for name, _, _, _, unexpected_successes in results
+        if unexpected_successes > 0]
 
     pass_count = sum([result[2] for result in results])
     fail_count = sum([result[3] for result in results])
 
-    return (timed_out, passed, failed, unexpected_passes, pass_count, fail_count)
+    return (
+        timed_out, passed, failed, unexpected_passes, pass_count, fail_count)
 
 in_q = None
 out_q = None
@@ -1510,6 +1515,16 @@ def main(num_threads, test_subdir, test_runner_name, results_formatter):
     (timed_out, passed, failed, unexpected_successes, pass_count,
      fail_count) = summary_results
 
+    # Check if we have any tests to rerun.
+    if results_formatter is not None:
+        tests_for_rerun = results_formatter.tests_for_rerun
+        results_formatter.tests_for_rerun = None
+
+        if tests_for_rerun is not None and len(tests_for_rerun) > 0:
+            # Here's where we trigger the re-run in a future change.
+            # Make sure the rest of the changes don't break anything.
+            pass
+
     # The results formatter - if present - is done now.  Tell it to
     # terminate.
     if results_formatter is not None:
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index ed01eccf238..3063a91407b 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -393,6 +393,9 @@ def parseOptionsAndInitTestdirs():
         configuration.results_formatter_name = (
             "lldbsuite.test.basic_results_formatter.BasicResultsFormatter")
 
+    # rerun-related arguments
+    configuration.rerun_all_issues = args.rerun_all_issues
+
     if args.lldb_platform_name:
         configuration.lldb_platform_name = args.lldb_platform_name
     if args.lldb_platform_url:
diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py
index ddb615ef5df..38f65b2521d 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest_args.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py
@@ -160,6 +160,15 @@ def create_parser():
               'pairs to all test events generated by this test run.  VAL may '
               'be specified as VAL:TYPE, where TYPE may be int to convert '
               'the value to an int'))
+
+    group = parser.add_argument_group('Test Re-run Options')
+    group.add_argument(
+        '--rerun-all-issues',
+        action='store_true',
+        help=('Re-run all issues that occurred during the test run '
+              'irrespective of the test method\'s marking as flakey. '
+              'Default behavior is to apply re-runs only to flakey '
+              'tests that generate issues.'))
     # Remove the reference to our helper function
     del X
 
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park b/lldb/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park
new file mode 100644
index 00000000000..7a5b2ba99f4
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park
@@ -0,0 +1,24 @@
+"""Tests that an exceptional exit is detected by the testbot."""
+
+from __future__ import print_function
+
+import atexit
+import os
+import signal
+import time
+
+import lldbsuite.test.lldbtest as lldbtest
+
+
+class ExceptionalExitOutOfTestMethodTestCase(lldbtest.TestBase):
+    """Forces exceptional exit."""
+    mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+    @lldbtest.skipIfWindows
+    def test_buildbot_catches_exceptional_exit(self):
+        pass
+
+def cleanup():
+        os.kill(os.getpid(), signal.SIGKILL)
+
+atexit.register(cleanup)
diff --git a/lldb/packages/Python/lldbsuite/test/result_formatter.py b/lldb/packages/Python/lldbsuite/test/result_formatter.py
index cf0a6cf6b31..ff77c1411ba 100644
--- a/lldb/packages/Python/lldbsuite/test/result_formatter.py
+++ b/lldb/packages/Python/lldbsuite/test/result_formatter.py
@@ -28,6 +28,7 @@ import six
 from six.moves import cPickle
 
 # LLDB modules
+from . import configuration
 
 
 # Ignore method count on DTOs.
@@ -165,7 +166,8 @@ class EventBuilder(object):
 
     RESULT_TYPES = set([
         TYPE_JOB_RESULT,
-        TYPE_TEST_RESULT])
+        TYPE_TEST_RESULT
+        ])
 
     # Test/Job Status Tags
     STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
@@ -178,6 +180,16 @@ class EventBuilder(object):
     STATUS_ERROR = "error"
     STATUS_TIMEOUT = "timeout"
 
+    """Test methods or jobs with a status matching any of these
+    status values will cause a testrun failure, unless
+    the test methods rerun and do not trigger an issue when rerun."""
+    TESTRUN_ERROR_STATUS_VALUES = set([
+        STATUS_ERROR,
+        STATUS_EXCEPTIONAL_EXIT,
+        STATUS_FAILURE,
+        STATUS_TIMEOUT
+        ])
+
     @staticmethod
     def _get_test_name_info(test):
         """Returns (test-class-name, test-method-name) from a test case instance.
@@ -655,6 +667,61 @@ class ResultsFormatter(object):
         # timeout test status for this.
         self.expected_timeouts_by_basename = set()
 
+        # Keep track of rerun-eligible tests.
+        # This is a set that contains tests saved as:
+        # {test_filename}:{test_class}:{test_name}
+        self.rerun_eligible_tests = set()
+
+        # A dictionary of test files that had a failing
+        # test, in the format of:
+        # key = test path, value = array of test methods that need rerun
+        self.tests_for_rerun = {}
+
+    @classmethod
+    def _make_rerun_eligibility_key(cls, test_result_event):
+        if test_result_event is None:
+            return None
+        component_count = 0
+        if "test_filename" in test_result_event:
+            key = test_result_event["test_filename"]
+            component_count += 1
+        if "test_class" in test_result_event:
+            if component_count > 0:
+                key += "."
+            key += test_result_event["test_class"]
+            component_count += 1
+        if "test_name" in test_result_event:
+            if component_count > 0:
+                key += "."
+            key += test_result_event["test_name"]
+            component_count += 1
+        return key
+
+    def _mark_test_for_rerun_eligibility(self, test_result_event):
+        key = self._make_rerun_eligibility_key(test_result_event)
+        if key is not None:
+            self.rerun_eligible_tests.add(key)
+        else:
+            sys.stderr.write(
+                "\nerror: test marked for re-run eligibility but "
+                "failed to create key.\n")
+
+    def _maybe_add_test_to_rerun_list(self, result_event):
+        key = self._make_rerun_eligibility_key(result_event)
+        if key is not None:
+            if key in self.rerun_eligible_tests or configuration.rerun_all_issues:
+                test_filename = result_event.get("test_filename", None)
+                if test_filename is not None:
+                    test_name = result_event.get("test_name", None)
+                    if not test_filename in self.tests_for_rerun:
+                        self.tests_for_rerun[test_filename] = []
+                    if test_name is not None:
+                        self.tests_for_rerun[test_filename].append(test_name)
+        else:
+            sys.stderr.write(
+                "\nerror: couldn't add testrun-failing test to rerun "
+                "list because no eligibility key could be created.\n")
+
     def _maybe_remap_job_result_event(self, test_event):
         """Remaps timeout/exceptional exit job results to last test method running.
 
@@ -684,13 +751,6 @@ class ResultsFormatter(object):
                 if start_key not in test_event:
                     test_event[start_key] = start_value
 
-            # Always take the value of test_filename from test_start,
-            # as it was gathered by class introspections.  Job status
-            # has less refined info available to it, so might be missing
-            # path info.
-            if "test_filename" in test_start:
-                test_event["test_filename"] = test_start["test_filename"]
-
     def _maybe_remap_expected_timeout(self, event):
         if event is None:
             return
@@ -749,12 +809,21 @@ class ResultsFormatter(object):
                 worker_index = test_event.get("worker_index", None)
                 if worker_index is not None:
                     self.started_tests_by_worker.pop(worker_index, None)
+
+                if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
+                    # A test/job status value in any of those status values
+                    # causes a testrun failure.  If such a test fails, check
+                    # whether it can be rerun.  If it can be rerun, add it
+                    # to the rerun job.
+                    self._maybe_add_test_to_rerun_list(test_event)
             elif event_type == EventBuilder.TYPE_TEST_START:
                 # Keep track of the most recent test start event
                 # for the related worker.
                 worker_index = test_event.get("worker_index", None)
                 if worker_index is not None:
                     self.started_tests_by_worker[worker_index] = test_event
+            elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
+                self._mark_test_for_rerun_eligibility(test_event)
 
     def set_expected_timeouts_by_basename(self, basenames):
         """Specifies a list of test file basenames that are allowed to timeout