Add expected timeout support to test event architecture.

llvm-svn: 255363

4 files changed, 74 insertions, 1 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py b/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
index a9e85dca90e..aded67532a3 100644
--- a/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
+++ b/lldb/packages/Python/lldbsuite/test/basic_results_formatter.py
@@ -315,6 +315,11 @@ class BasicResultsFormatter(result_formatter.ResultsFormatter):
             [result_formatter.EventBuilder.STATUS_SKIP, "Skip", False, None],
             [result_formatter.EventBuilder.STATUS_TIMEOUT,
              "Timeout", True, "TIMEOUT"],
+            [result_formatter.EventBuilder.STATUS_EXPECTED_TIMEOUT,
+             # Intentionally using the unusual hyphenation in TIME-OUT to
+             # prevent buildbots from thinking it is an issue when scanning
+             # for TIMEOUT.
+             "Expected Timeout", True, "EXPECTED TIME-OUT"]
             ]
 
         # Partition all the events by test result status
diff --git a/lldb/packages/Python/lldbsuite/test/dosep.py b/lldb/packages/Python/lldbsuite/test/dosep.py
index 3f7b92e30fd..03524962c95 100644
--- a/lldb/packages/Python/lldbsuite/test/dosep.py
+++ b/lldb/packages/Python/lldbsuite/test/dosep.py
@@ -1128,6 +1128,7 @@ def getExpectedTimeouts(platform_name):
         target = m.group(1)
 
     expected_timeout = set()
+    expected_timeout.add("TestExpectedTimeout.py")
 
     if target.startswith("linux"):
         expected_timeout |= {
@@ -1475,6 +1476,12 @@ def main(num_threads, test_subdir, test_runner_name, results_formatter):
 
     system_info = " ".join(platform.uname())
 
+    # Figure out which test files should be enabled for expected
+    # timeout
+    expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
+    if results_formatter is not None:
+        results_formatter.set_expected_timeouts_by_basename(expected_timeout)
+
     # Figure out which testrunner strategy we'll use.
     runner_strategies_by_name = get_test_runner_strategies(num_threads)
 
@@ -1514,7 +1521,6 @@ def main(num_threads, test_subdir, test_runner_name, results_formatter):
         os.rename(core, os.path.join(session_dir, dst))
 
     # remove expected timeouts from failures
-    expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
     for xtime in expected_timeout:
         if xtime in timed_out:
             timed_out.remove(xtime)
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park b/lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
new file mode 100644
index 00000000000..2453581231f
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
@@ -0,0 +1,19 @@
+"""Tests that a timeout is detected by the testbot."""
+from __future__ import print_function
+
+import time
+
+import lldbsuite.test.lldbtest as lldbtest
+
+
+class ExpectedTimeoutTestCase(lldbtest.TestBase):
+    """Forces test timeout."""
+    mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+    def test_buildbot_sees_expected_timeout(self):
+        """Tests that expected timeout logic kicks in and is picked up."""
+        while True:
+            try:
+                time.sleep(1)
+            except:
+                print("ignoring exception during sleep")
diff --git a/lldb/packages/Python/lldbsuite/test/result_formatter.py b/lldb/packages/Python/lldbsuite/test/result_formatter.py
index e05714f384b..9cd37a049d0 100644
--- a/lldb/packages/Python/lldbsuite/test/result_formatter.py
+++ b/lldb/packages/Python/lldbsuite/test/result_formatter.py
@@ -163,11 +163,16 @@ class EventBuilder(object):
     TYPE_TEST_START = "test_start"
     TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
 
+    RESULT_TYPES = set([
+        TYPE_JOB_RESULT,
+        TYPE_TEST_RESULT])
+
     # Test/Job Status Tags
     STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
     STATUS_SUCCESS = "success"
     STATUS_FAILURE = "failure"
     STATUS_EXPECTED_FAILURE = "expected_failure"
+    STATUS_EXPECTED_TIMEOUT = "expected_timeout"
     STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
     STATUS_SKIP = "skip"
     STATUS_ERROR = "error"
@@ -622,6 +627,7 @@ class ResultsFormatter(object):
         self.result_status_counts = {
             EventBuilder.STATUS_SUCCESS: 0,
             EventBuilder.STATUS_EXPECTED_FAILURE: 0,
+            EventBuilder.STATUS_EXPECTED_TIMEOUT: 0,
             EventBuilder.STATUS_SKIP: 0,
             EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0,
             EventBuilder.STATUS_FAILURE: 0,
@@ -642,6 +648,13 @@ class ResultsFormatter(object):
         # entirely consistent from the outside.
         self.lock = threading.Lock()
 
+        # Keeps track of the test base filenames for tests that
+        # are expected to timeout.  If a timeout occurs in any test
+        # basename that matches this list, that result should be
+        # converted into a non-issue.  We'll create an expected
+        # timeout test status for this.
+        self.expected_timeouts_by_basename = set()
+
     def _maybe_remap_job_result_event(self, test_event):
         """Remaps timeout/exceptional exit job results to last test method running.
 
@@ -678,6 +691,21 @@ class ResultsFormatter(object):
             if "test_filename" in test_start:
                 test_event["test_filename"] = test_start["test_filename"]
 
+    def _maybe_remap_expected_timeout(self, event):
+        if event is None:
+            return
+
+        status = event.get("status", None)
+        if status is None or status != EventBuilder.STATUS_TIMEOUT:
+            return
+
+        # Check if the timeout test's basename is in the expected timeout
+        # list.  If so, convert to an expected timeout.
+        basename = os.path.basename(event.get("test_filename", ""))
+        if basename in self.expected_timeouts_by_basename:
+            # Convert to an expected timeout.
+            event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
+
     def handle_event(self, test_event):
         """Handles the test event for collection into the formatter output.
 
@@ -703,6 +731,11 @@ class ResultsFormatter(object):
                 self._maybe_remap_job_result_event(test_event)
                 event_type = test_event.get("event", "")
 
+            # Remap timeouts to expected timeouts.
+            if event_type in EventBuilder.RESULT_TYPES:
+                self._maybe_remap_expected_timeout(test_event)
+                event_type = test_event.get("event", "")
+
             if event_type == "terminate":
                 self.terminate_called = True
             elif (event_type == EventBuilder.TYPE_TEST_RESULT or
@@ -724,6 +757,16 @@ class ResultsFormatter(object):
                 if worker_index is not None:
                     self.started_tests_by_worker[worker_index] = test_event
 
+    def set_expected_timeouts_by_basename(self, basenames):
+        """Specifies a list of test file basenames that are allowed to timeout
+        without being called out as a timeout issue.
+
+        These fall into a new status category called STATUS_EXPECTED_TIMEOUT.
+        """
+        if basenames is not None:
+            for basename in basenames:
+                self.expected_timeouts_by_basename.add(basename)
+
     def track_start_time(self, test_class, test_name, start_time):
         """tracks the start time of a test so elapsed time can be computed.