Add TestConcurrentEvents test for LLDB's handling of inferior threads

- Test verifies LLDB's handling of inferiors with threads that: hit breakpoints,
  modfiy variables that have watchpoints set, generate user signals, and crash.
- Add a few "stress tests" (with ~100 threads) -- run these with "-l" dotest.py
  flag.
- Fix stop_reason_to_str helper in lldbutil to handle eStopReasonThreadExited.
- Add sort_stopped_threads helper to lldbutil to separate thread lists based
  on stop reason.

Logged llvm.org/pr16566 and llvm.org/pr16567 for bugs exposed.

llvm-svn: 185889

6 files changed, 746 insertions, 0 deletions

diff --git a/lldb/test/functionalities/thread/concurrent_events/Makefile b/lldb/test/functionalities/thread/concurrent_events/Makefile
new file mode 100644
index 00000000000..89c0e817259
--- /dev/null
+++ b/lldb/test/functionalities/thread/concurrent_events/Makefile
@@ -0,0 +1,8 @@
+LEVEL = ../../../make
+
+CXX_SOURCES := main.cpp
+
+CFLAGS_EXTRAS += -std=c++11 -lpthread
+LD_EXTRAS += -lpthread
+
+include $(LEVEL)/Makefile.rules
diff --git a/lldb/test/functionalities/thread/concurrent_events/TestConcurrentEvents.py b/lldb/test/functionalities/thread/concurrent_events/TestConcurrentEvents.py
new file mode 100644
index 00000000000..88055e9285a
--- /dev/null
+++ b/lldb/test/functionalities/thread/concurrent_events/TestConcurrentEvents.py
@@ -0,0 +1,485 @@
+"""
+A stress-test of sorts for LLDB's handling of threads in the inferior.
+
+This test sets a breakpoint in the main thread where test parameters (numbers of
+threads) can be adjusted, runs the inferior to that point, and modifies the
+locals that control the event thread counts. This test also sets a breakpoint in
+breakpoint_func (the function executed by each 'breakpoint' thread) and a
+watchpoint on a global modified in watchpoint_func. The inferior is continued
+until exit or a crash takes place, and the number of events seen by LLDB is
+verified to match the expected number of events.
+"""
+
+import os, time
+import unittest2
+import lldb
+from lldbtest import *
+import lldbutil
+
+class ConcurrentEventsTestCase(TestBase):
+
+    mydir = os.path.join("functionalities", "thread", "concurrent_events")
+
+    #
+    ## Tests for multiple threads that generate a single event.
+    #
+    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
+    @dwarf_test
+    def test_many_breakpoints_dwarf(self):
+        """Test 100 breakpoints from 100 threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=100)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
+    @dwarf_test
+    def test_many_watchpoints_dwarf(self):
+        """Test 100 watchpoints from 100 threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=100)
+
+    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @dwarf_test
+    def test_many_signals_dwarf(self):
+        """Test 100 signals from 100 threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_signal_threads=100)
+
+    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
+    @dwarf_test
+    def test_many_crash_dwarf(self):
+        """Test 100 threads that cause a segfault."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_crash_threads=100)
+
+
+    #
+    ## Tests for concurrent signal and breakpoint
+    #
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_break_dwarf(self):
+        """Test signal and a breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1, num_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_delay_signal_break_dwarf(self):
+        """Test (1-second delay) signal and a breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1, num_delay_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_delay_break_dwarf(self):
+        """Test signal and a (1 second delay) breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_delay_breakpoint_threads=1, num_signal_threads=1)
+
+
+    #
+    ## Tests for concurrent watchpoint and breakpoint
+    #
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_watch_break_dwarf(self):
+        """Test watchpoint and a breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1, num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_delay_watch_break_dwarf(self):
+        """Test (1-second delay) watchpoint and a breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1, num_delay_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_watch_break_dwarf(self):
+        """Test watchpoint and a (1 second delay) breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_delay_breakpoint_threads=1, num_watchpoint_threads=1)
+
+    #
+    ## Tests for concurrent signal and watchpoint
+    #
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_watch_dwarf(self):
+        """Test a watchpoint and a signal in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_signal_threads=1, num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_delay_signal_watch_dwarf(self):
+        """Test a watchpoint and a (1 second delay) signal in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_delay_signal_threads=1, num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_delay_watch_dwarf(self):
+        """Test a (1 second delay) watchpoint and a signal in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_signal_threads=1, num_delay_watchpoint_threads=1)
+
+
+    #
+    ## Tests for multiple breakpoint threads
+    #
+    @dwarf_test
+    def test_two_breakpoint_threads_dwarf(self):
+        """Test two threads that trigger a breakpoint. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=2)
+
+    @dwarf_test
+    def test_breakpoint_one_delay_breakpoint_threads_dwarf(self):
+        """Test threads that trigger a breakpoint where one thread has a 1 second delay. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1,
+                               num_delay_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_two_breakpoints_one_signal_dwarf(self):
+        """Test two threads that trigger a breakpoint and one signal thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=2, num_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_breakpoint_delay_breakpoint_one_signal_dwarf(self):
+        """Test two threads that trigger a breakpoint (one with a 1 second delay) and one signal thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1,
+                               num_delay_breakpoint_threads=1,
+                               num_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_two_breakpoints_one_delay_signal_dwarf(self):
+        """Test two threads that trigger a breakpoint and one (1 second delay) signal thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=2, num_delay_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_two_breakpoints_one_watchpoint_dwarf(self):
+        """Test two threads that trigger a breakpoint and one watchpoint thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=2, num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_breakpoints_delayed_breakpoint_one_watchpoint_dwarf(self):
+        """Test a breakpoint, a delayed breakpoint, and one watchpoint thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_breakpoint_threads=1,
+                               num_delay_breakpoint_threads=1,
+                               num_watchpoint_threads=1)
+
+    #
+    ## Tests for multiple watchpoint threads
+    #
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_two_watchpoint_threads_dwarf(self):
+        """Test two threads that trigger a watchpoint. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=2)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_watchpoint_with_delay_waychpoint_threads_dwarf(self):
+        """Test two threads that trigger a watchpoint where one thread has a 1 second delay. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=1,
+                               num_delay_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_two_watchpoints_one_breakpoint_dwarf(self):
+        """Test two threads that trigger a watchpoint and one breakpoint thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=2, num_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_two_watchpoints_one_delay_breakpoint_dwarf(self):
+        """Test two threads that trigger a watchpoint and one (1 second delay) breakpoint thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=2, num_delay_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_watchpoint_delay_watchpoint_one_breakpoint_dwarf(self):
+        """Test two threads that trigger a watchpoint (one with a 1 second delay) and one breakpoint thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=1,
+                               num_delay_watchpoint_threads=1,
+                               num_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_two_watchpoints_one_signal_dwarf(self):
+        """Test two threads that trigger a watchpoint and one signal thread. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=2, num_signal_threads=1)
+
+    #
+    ## Test for watchpoint, signal and breakpoint happening concurrently
+    #
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_watch_break_dwarf(self):
+        """Test a signal/watchpoint/breakpoint in multiple threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_signal_threads=1,
+                               num_watchpoint_threads=1,
+                               num_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_signal_watch_break_dwarf(self):
+        """Test one signal thread with 5 watchpoint and breakpoint threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_signal_threads=1,
+                               num_watchpoint_threads=5,
+                               num_breakpoint_threads=5)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_signal_watch_break_dwarf(self):
+        """Test with 5 watchpoint and breakpoint threads."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_watchpoint_threads=5,
+                               num_breakpoint_threads=5)
+
+
+    #
+    ## Test for crashing threads happening concurrently with other events
+    #
+    @dwarf_test
+    def test_crash_with_break_dwarf(self):
+        """ Test a thread that crashes while another thread hits a breakpoint."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_crash_threads=1, num_breakpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @dwarf_test
+    def test_crash_with_watchpoint_dwarf(self):
+        """ Test a thread that crashes while another thread hits a watchpoint."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_crash_threads=1, num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_crash_with_signal_dwarf(self):
+        """ Test a thread that crashes while another thread generates a signal."""
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_crash_threads=1, num_signal_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_crash_with_watchpoint_breakpoint_signal_dwarf(self):
+        """ Test a thread that crashes while other threads generate a signal and hit a watchpoint and breakpoint. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_crash_threads=1,
+                               num_breakpoint_threads=1,
+                               num_signal_threads=1,
+                               num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16566 -- new threads do not respect watchpoints
+    @skipIfLinux # inferior thread enters state D (disk sleep) which causes hang in lldb
+    @dwarf_test
+    def test_delayed_crash_with_breakpoint_watchpoint_dwarf(self):
+        """ Test a thread with a delayed crash while other threads hit a watchpoint and a breakpoint. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_delay_crash_threads=1,
+                               num_breakpoint_threads=1,
+                               num_watchpoint_threads=1)
+
+    @skipIfDarwin # llvm.org/pr16567 -- thread count is incorrect during signal delivery
+    @skipIfLinux # llvm.org/pr16567 -- explicit signals cause hangs on Linux
+    @dwarf_test
+    def test_delayed_crash_with_breakpoint_signal_dwarf(self):
+        """ Test a thread with a delayed crash while other threads generate a signal and hit a breakpoint. """
+        self.buildDwarf(dictionary=self.getBuildFlags())
+        self.do_thread_actions(num_delay_crash_threads=1,
+                               num_breakpoint_threads=1,
+                               num_signal_threads=1)
+
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Find the line number for our breakpoint.
+        self.thread_breakpoint = line_number('main.cpp', '// Set breakpoint here')
+        self.setup_breakpoint = line_number('main.cpp', '// Break here and adjust num')
+
+    def print_threads(self, threads):
+        ret = ""
+        for x in threads:
+            ret += "\t thread %d stopped due to reason %s" % (x.GetIndexID(), lldbutil.stop_reason_to_str(x.GetStopReason()))
+        return ret
+
+    def debug_threads(self, bps, crashed, exiting, wps, signals, others):
+        print "%d threads stopped at bp:\n%s" % (len(bps), self.print_threads(bps))
+        print "%d threads crashed:\n%s" % (len(crashed), self.print_threads(crashed))
+        print "%d threads stopped due to watchpoint:\n%s" % (len(wps), self.print_threads(wps))
+        print "%d threads stopped at signal:\n%s" % (len(signals), self.print_threads(signals))
+        print "%d threads exiting:\n%s" % (len(exiting), self.print_threads(exiting))
+        print "%d threads stopped due to other/unknown reason:\n%s" % (len(others), self.print_threads(others))
+
+    def do_thread_actions(self,
+                          num_breakpoint_threads = 0,
+                          num_signal_threads = 0,
+                          num_watchpoint_threads = 0,
+                          num_crash_threads = 0,
+                          num_delay_breakpoint_threads = 0,
+                          num_delay_signal_threads = 0,
+                          num_delay_watchpoint_threads = 0,
+                          num_delay_crash_threads = 0):
+        """ Sets a breakpoint in the main thread where test parameters (numbers of threads) can be adjusted, runs the inferior
+            to that point, and modifies the locals that control the event thread counts. Also sets a breakpoint in
+            breakpoint_func (the function executed by each 'breakpoint' thread) and a watchpoint on a global modified in
+            watchpoint_func. The inferior is continued until exit or a crash takes place, and the number of events seen by LLDB
+            is verified to match the expected number of events.
+        """
+        exe = os.path.join(os.getcwd(), "a.out")
+        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
+
+        # Initialize all the breakpoints (main thread/aux thread)
+        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.setup_breakpoint,
+            num_expected_locations=1)
+        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.thread_breakpoint,
+            num_expected_locations=1)
+
+        # The breakpoint list should show 2 breakpoints with 1 location.
+        self.expect("breakpoint list -f", "Breakpoint location shown correctly",
+            substrs = ["1: file = 'main.cpp', line = %d, locations = 1" % self.setup_breakpoint,
+                       "2: file = 'main.cpp', line = %d, locations = 1" % self.thread_breakpoint])
+
+        # Run the program.
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        # Check we are at line self.setup_breakpoint
+        self.expect("thread backtrace", STOPPED_DUE_TO_BREAKPOINT,
+            substrs = ["stop reason = breakpoint 1."])
+
+        # Initialize the watchpoint on the global variable (g_watchme)
+        if num_watchpoint_threads + num_delay_watchpoint_threads > 0:
+            self.runCmd("watchpoint set variable g_watchme")
+
+        # Get the target process
+        target = self.dbg.GetSelectedTarget()
+        process = target.GetProcess()
+
+        # We should be stopped at the setup site where we can set the number of
+        # threads doing each action (break/crash/signal/watch)
+        self.assertEqual(process.GetNumThreads(), 1, 'Expected to stop before any additional threads are spawned.')
+
+        self.runCmd("expr num_breakpoint_threads=%d" % num_breakpoint_threads)
+        self.runCmd("expr num_crash_threads=%d" % num_crash_threads)
+        self.runCmd("expr num_signal_threads=%d" % num_signal_threads)
+        self.runCmd("expr num_watchpoint_threads=%d" % num_watchpoint_threads)
+
+        self.runCmd("expr num_delay_breakpoint_threads=%d" % num_delay_breakpoint_threads)
+        self.runCmd("expr num_delay_crash_threads=%d" % num_delay_crash_threads)
+        self.runCmd("expr num_delay_signal_threads=%d" % num_delay_signal_threads)
+        self.runCmd("expr num_delay_watchpoint_threads=%d" % num_delay_watchpoint_threads)
+
+        self.runCmd("continue")
+
+        # Make sure we see all the threads. The inferior program's threads all synchronize with a pseudo-barrier; that is,
+        # the inferior program ensures all threads are started and running before any thread triggers its 'event'.
+        num_threads = process.GetNumThreads()
+        expected_num_threads = num_breakpoint_threads + num_delay_breakpoint_threads \
+                             + num_signal_threads + num_delay_signal_threads \
+                             + num_watchpoint_threads + num_delay_watchpoint_threads \
+                             + num_crash_threads + num_delay_crash_threads + 1
+        self.assertEqual(num_threads, expected_num_threads,
+            'Number of expected threads and actual threads do not match.')
+
+        # Get the thread objects
+        (breakpoint_threads, crashed_threads, exiting_threads, other_threads, signal_threads, watchpoint_threads) = ([], [], [], [], [], [])
+        lldbutil.sort_stopped_threads(process,
+                                      breakpoint_threads=breakpoint_threads,
+                                      crashed_threads=crashed_threads,
+                                      exiting_threads=exiting_threads,
+                                      signal_threads=signal_threads,
+                                      watchpoint_threads=watchpoint_threads,
+                                      other_threads=other_threads)
+
+        if self.TraceOn():
+            self.debug_threads(breakpoint_threads, crashed_threads, exiting_threads, watchpoint_threads, signal_threads, other_threads)
+
+        # The threads that are doing signal handling must be unblocked or the inferior will hang. We keep
+        # a counter of threads that stop due to a signal so we have something to verify later on.
+        seen_signal_threads = len(signal_threads)
+        seen_breakpoint_threads = len(breakpoint_threads)
+        seen_watchpoint_threads = len(watchpoint_threads)
+        seen_crashed_threads = len(crashed_threads)
+
+        # Run to completion
+        while len(crashed_threads) == 0 and process.GetState() != lldb.eStateExited:
+            if self.TraceOn():
+                self.runCmd("thread backtrace all")
+                self.debug_threads(breakpoint_threads, crashed_threads, exiting_threads, watchpoint_threads, signal_threads, other_threads)
+
+            self.runCmd("continue")
+            lldbutil.sort_stopped_threads(process,
+                                          breakpoint_threads=breakpoint_threads,
+                                          crashed_threads=crashed_threads,
+                                          exiting_threads=exiting_threads,
+                                          signal_threads=signal_threads,
+                                          watchpoint_threads=watchpoint_threads,
+                                          other_threads=other_threads)
+            seen_signal_threads += len(signal_threads)
+            seen_breakpoint_threads += len(breakpoint_threads)
+            seen_watchpoint_threads += len(watchpoint_threads)
+            seen_crashed_threads += len(crashed_threads)
+
+        if num_crash_threads > 0 or num_delay_crash_threads > 0:
+            # Expecting a crash
+            self.assertTrue(seen_crashed_threads > 0, "Expecting at least one thread to crash")
+
+            # Ensure the zombie process is reaped
+            self.runCmd("process kill")
+
+        elif num_crash_threads == 0 and num_delay_crash_threads == 0:
+            # The inferior process should have exited without crashing
+            self.assertEqual(0, seen_crashed_threads, "Unexpected thread(s) in crashed state")
+            self.assertTrue(process.GetState() == lldb.eStateExited, PROCESS_EXITED)
+
+            # Verify the number of actions took place matches expected numbers
+            self.assertEqual(num_delay_breakpoint_threads + num_breakpoint_threads, seen_breakpoint_threads)
+            self.assertEqual(num_delay_signal_threads + num_signal_threads, seen_signal_threads)
+            self.assertEqual(num_delay_watchpoint_threads + num_watchpoint_threads, seen_watchpoint_threads)
+
+
+if __name__ == '__main__':
+    import atexit
+    lldb.SBDebugger.Initialize()
+    atexit.register(lambda: lldb.SBDebugger.Terminate())
+    unittest2.main()
diff --git a/lldb/test/functionalities/thread/concurrent_events/main.cpp b/lldb/test/functionalities/thread/concurrent_events/main.cpp
new file mode 100644
index 00000000000..ffda2c35876
--- /dev/null
+++ b/lldb/test/functionalities/thread/concurrent_events/main.cpp
@@ -0,0 +1,199 @@
+//===-- main.cpp ------------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// This test is intended to create a situation in which a watchpoint will be hit
+// while a breakpoint is being handled in another thread.  The expected result is
+// that the watchpoint in the second thread will be hit while the breakpoint handler
+// in the first thread is trying to stop all threads.
+
+#include <atomic>
+#include <vector>
+using namespace std;
+
+#include <pthread.h>
+#include <semaphore.h>
+
+#include <signal.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+// Note that although hogging the CPU while waiting for a variable to change
+// would be terrible in production code, it's great for testing since it
+// avoids a lot of messy context switching to get multiple threads synchronized.
+#define do_nothing()
+
+#define pseudo_barrier_wait(bar) \
+    --bar;                       \
+    while (bar > 0)              \
+        do_nothing();
+
+#define pseudo_barrier_init(bar, count) (bar = count)
+
+typedef std::vector<std::pair<unsigned, void*(*)(void*)> > action_counts;
+typedef std::vector<pthread_t> thread_vector;
+
+std::atomic_int g_barrier;
+int g_breakpoint = 0;
+int g_sigusr1_count = 0;
+std::atomic_int g_watchme;
+
+//sem_t g_signal_semaphore;
+
+struct action_args {
+  int delay;
+};
+
+// Perform any extra actions required by thread 'input' arg
+void do_action_args(void *input) {
+    if (input) {
+      action_args *args = static_cast<action_args*>(input);
+      sleep(args->delay);
+    }
+}
+
+void *
+breakpoint_func (void *input)
+{
+    // Wait until both threads are running
+    pseudo_barrier_wait(g_barrier);
+    do_action_args(input);
+
+    // Do something
+    g_breakpoint++;       // Set breakpoint here
+    return 0;
+}
+
+void *
+signal_func (void *input) {
+    // Wait until both threads are running
+    pseudo_barrier_wait(g_barrier);
+    do_action_args(input);
+
+    // Generate a user-defined signal to current process
+    kill(getpid(), SIGUSR1);
+
+    // wait for notification the signal handler was executed
+    //sem_wait(&g_signal_semaphore);
+
+    return 0;
+}
+
+void *
+watchpoint_func (void *input) {
+    pseudo_barrier_wait(g_barrier);
+    do_action_args(input);
+
+    g_watchme += 1;     // watchpoint triggers here
+    return 0;
+}
+
+void *
+crash_func (void *input) {
+    pseudo_barrier_wait(g_barrier);
+    do_action_args(input);
+
+    int *a = 0;
+    *a = 5; // crash happens here
+    return 0;
+}
+
+void sigusr1_handler(int sig) {
+    //sem_post(&g_signal_semaphore);
+    if (sig == SIGUSR1)
+        g_sigusr1_count += 1; // Break here in signal handler
+}
+
+/// Register a simple function for to handle signal
+void register_signal_handler(int signal, void (*handler)(int))
+{
+    sigset_t empty_sigset;
+    sigemptyset(&empty_sigset);
+
+    struct sigaction action;
+    action.sa_sigaction = 0;
+    action.sa_mask = empty_sigset;
+    action.sa_flags = 0;
+    action.sa_handler = handler;
+    sigaction(SIGUSR1, &action, 0);
+
+    //sem_init(&g_signal_semaphore, 0, 0);
+}
+
+void start_threads(thread_vector& threads,
+                   action_counts& actions,
+                   void* args = 0) {
+    action_counts::iterator b = actions.begin(), e = actions.end();
+    for(action_counts::iterator i = b; i != e; ++i) {
+        for(unsigned count = 0; count < i->first; ++count) {
+            pthread_t t;
+            pthread_create(&t, 0, i->second, args);
+            threads.push_back(t);
+        }
+    }
+}
+
+int main ()
+{
+    g_watchme = 0;
+
+    unsigned num_breakpoint_threads = 1;
+    unsigned num_watchpoint_threads = 0;
+    unsigned num_signal_threads = 0;
+    unsigned num_crash_threads = 1;
+
+    unsigned num_delay_breakpoint_threads = 0;
+    unsigned num_delay_watchpoint_threads = 0;
+    unsigned num_delay_signal_threads = 0;
+    unsigned num_delay_crash_threads = 0;
+
+    // Break here and adjust num_[breakpoint|watchpoint|signal|crash]_threads
+    unsigned total_threads = num_breakpoint_threads \
+                             + num_watchpoint_threads \
+                             + num_signal_threads \
+                             + num_crash_threads \
+                             + num_delay_breakpoint_threads \
+                             + num_delay_watchpoint_threads \
+                             + num_delay_signal_threads \
+                             + num_delay_crash_threads;
+
+    // Don't let either thread do anything until they're both ready.
+    pseudo_barrier_init(g_barrier, total_threads);
+
+    thread_vector threads;
+
+    action_counts actions;
+    actions.push_back(std::make_pair(num_breakpoint_threads, breakpoint_func));
+    actions.push_back(std::make_pair(num_watchpoint_threads, watchpoint_func));
+    actions.push_back(std::make_pair(num_signal_threads, signal_func));
+    actions.push_back(std::make_pair(num_crash_threads, crash_func));
+
+    action_counts delay_actions;
+    actions.push_back(std::make_pair(num_delay_breakpoint_threads, breakpoint_func));
+    actions.push_back(std::make_pair(num_delay_watchpoint_threads, watchpoint_func));
+    actions.push_back(std::make_pair(num_delay_signal_threads, signal_func));
+    actions.push_back(std::make_pair(num_delay_crash_threads, crash_func));
+
+    register_signal_handler(SIGUSR1, sigusr1_handler);
+
+    // Create threads that handle instant actions
+    start_threads(threads, actions);
+
+    // Create threads that handle delayed actions
+    action_args delay_arg;
+    delay_arg.delay = 1;
+    start_threads(threads, delay_actions, &delay_arg);
+
+    // Join all threads
+    typedef std::vector<pthread_t>::iterator thread_iterator;
+    for(thread_iterator t = threads.begin(); t != threads.end(); ++t)
+        pthread_join(*t, 0);
+
+    // Break here and verify one thread is active.
+    return 0;
+}
diff --git a/lldb/test/functionalities/watchpoint/multiple_threads/TestWatchpointMultipleThreads.py b/lldb/test/functionalities/watchpoint/multiple_threads/TestWatchpointMultipleThreads.py
index 2bf5d8403b6..f4fc0c43729 100644
--- a/lldb/test/functionalities/watchpoint/multiple_threads/TestWatchpointMultipleThreads.py
+++ b/lldb/test/functionalities/watchpoint/multiple_threads/TestWatchpointMultipleThreads.py
@@ -65,6 +65,7 @@ class WatchpointForMultipleThreadsTestCase(TestBase):
         # Add a breakpoint to set a watchpoint when stopped on the breakpoint.
         lldbutil.run_break_set_by_file_and_line (self, None, self.first_stop, num_expected_locations=1)
 
+        # llvm.org/pr16566: LLDB requires a breakpoint to be hit before watchpoints are respected on a thread created after the watchpoing is set.
         # Set this breakpoint to allow newly created thread to inherit the global watchpoint state.
         lldbutil.run_break_set_by_file_and_line (self, None, self.thread_function, num_expected_locations=1)
 
diff --git a/lldb/test/lldbtest.py b/lldb/test/lldbtest.py
index ba1b1d9fb32..679eafed33e 100644
--- a/lldb/test/lldbtest.py
+++ b/lldb/test/lldbtest.py
@@ -618,6 +618,22 @@ def skipIfLinux(func):
             func(*args, **kwargs)
     return wrapper
 
+def skipIfDarwin(func):
+    """Decorate the item to skip tests that should be skipped on Darwin."""
+    if isinstance(func, type) and issubclass(func, unittest2.TestCase):
+        raise Exception("@skipIfLinux can only be used to decorate a test method")
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        from unittest2 import case
+        self = args[0]
+        platform = sys.platform
+        if "darwin" in platform:
+            self.skipTest("skip on darwin")
+        else:
+            func(*args, **kwargs)
+    return wrapper
+
+
 def skipIfLinuxClang(func):
     """Decorate the item to skip tests that should be skipped if building on 
        Linux with clang.
diff --git a/lldb/test/lldbutil.py b/lldb/test/lldbutil.py
index 76e4e297a44..a7fcee8a16a 100644
--- a/lldb/test/lldbutil.py
+++ b/lldb/test/lldbutil.py
@@ -181,6 +181,8 @@ def stop_reason_to_str(enum):
         return "exception"
     elif enum == lldb.eStopReasonPlanComplete:
         return "plancomplete"
+    elif enum == lldb.eStopReasonThreadExiting:
+        return "threadexiting"
     else:
         raise Exception("Unknown StopReason enum")
 
@@ -258,6 +260,41 @@ def value_type_to_str(enum):
 
 
 # ==================================================
+# Get stopped threads due to each stop reason.
+# ==================================================
+
+def sort_stopped_threads(process,
+                         breakpoint_threads = None,
+                         crashed_threads = None,
+                         watchpoint_threads = None,
+                         signal_threads = None,
+                         exiting_threads = None,
+                         other_threads = None):
+    """ Fills array *_threads with threads stopped for the corresponding stop
+        reason.
+    """
+    for lst in [breakpoint_threads,
+                watchpoint_threads,
+                signal_threads,
+                exiting_threads,
+                other_threads]:
+        if lst is not None:
+            lst[:] = []
+
+    for thread in process:
+        dispatched = False
+        for (reason, list) in [(lldb.eStopReasonBreakpoint, breakpoint_threads),
+                               (lldb.eStopReasonException, crashed_threads),
+                               (lldb.eStopReasonWatchpoint, watchpoint_threads),
+                               (lldb.eStopReasonSignal, signal_threads),
+                               (lldb.eStopReasonThreadExiting, exiting_threads),
+                               (None, other_threads)]:
+            if not dispatched and list is not None:
+                if thread.GetStopReason() == reason or reason is None:
+                    list.append(thread)
+                    dispatched = True
+
+# ==================================================
 # Utility functions for setting breakpoints
 # ==================================================