test infra: enable single-worker rerun phase for flakey tests.

Use of --rerun-all-issues will enable any test method failure, not just
test methods marked with the flakey decorator, to rerun.

Currently this does not change the flakey logic's immediate rerun
attempt.  I want to make sure this doesn't cause any significant issues
before changing that part.

The rerun reporting is only known to work properly with the
default (new) BasicResultsFormatter reporting.  Once we work out
any issues, I'll go back and make sure the curses output handles
it properly as well.

llvm-svn: 255543

3 files changed, 73 insertions, 0 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park b/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park
new file mode 100644
index 00000000000..bcd1926d740
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park
@@ -0,0 +1,23 @@
+"""Tests that a flakey fail is rerun, and will pass on the rerun.
+Run this test with --rerun-all-issues specified to test that
+the tests fail on the first run, then pass on the second.
+Do not mark them as flakey as, at this time, flakey tests will
+run twice, thus causing the second run to succeed."""
+
+from __future__ import print_function
+
+import rerun_base
+
+import lldbsuite.test.lldbtest as lldbtest
+
+
+class RerunFailTestCase(rerun_base.RerunBaseTestCase):
+    """Forces test failure on first run, success on rerun."""
+    @lldbtest.no_debug_info_test
+    def test_buildbot_catches_failure(self):
+        """Issues a failing test assertion."""
+        if self.should_generate_issue():
+            self.assertTrue(
+                False,
+                "This will fail on the first call, succeed on rerun, and "
+                "alternate thereafter.")
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park b/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park
new file mode 100644
index 00000000000..1cf5373ac49
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park
@@ -0,0 +1,22 @@
+"""Tests that a timeout is detected by the testbot."""
+from __future__ import print_function
+
+import time
+
+import lldbsuite.test.lldbtest as lldbtest
+import rerun_base
+
+
+class RerunTimeoutTestCase(rerun_base.RerunBaseTestCase):
+    @lldbtest.no_debug_info_test
+    def test_timeout_rerun_succeeds(self):
+        """Tests that timeout logic kicks in and is picked up."""
+        if not self.should_generate_issue():
+            # We pass this time.
+            return
+        # We time out this time.
+        while True:
+            try:
+                time.sleep(1)
+            except:
+                print("ignoring exception during sleep")
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/rerun_base.py b/lldb/packages/Python/lldbsuite/test/issue_verification/rerun_base.py
new file mode 100644
index 00000000000..2ce775dced1
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/issue_verification/rerun_base.py
@@ -0,0 +1,28 @@
+from __future__ import print_function
+
+import os
+
+import lldbsuite.test.lldbtest as lldbtest
+
+
+# pylint: disable=too-few-public-methods
+class RerunBaseTestCase(lldbtest.TestBase):
+    """Forces test failure."""
+    mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+    def should_generate_issue(self):
+        """Returns whether a test issue should be generated.
+
+        @returns True on the first and every other call via a given
+        test method.
+        """
+        should_pass_filename = "{}.{}.succeed-marker".format(
+            __file__, self.id())
+        fail = not os.path.exists(should_pass_filename)
+        if fail:
+            # Create the marker so that next call to this passes.
+            open(should_pass_filename, 'w').close()
+        else:
+            # Delete the marker so next time we fail.
+            os.remove(should_pass_filename)
+        return fail