Add "watch set" command as a more general interface in conjunction with "frame var -w".

Also add test cases for watching a variable as well as a location expressed as an expression.

o TestMyFirstWatchpoint.py:

  Modified to test "watchpoint set -w write global".

o TestWatchLocationWithWatchSet.py:

  Added to test "watchpoint set -w write -x 1 g_char_ptr + 7" where a contrived example program
  with several threads is supposed to only access the array index within the range [0..6], but
  there's some misbehaving thread writing past the range.

rdar://problem/10701761

llvm-svn: 149280

7 files changed, 473 insertions, 8 deletions

diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index 0b18e9abe8e..ae2cdd92121 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -122,7 +122,7 @@ Watchpoint::DumpWithLevel(Stream *s, lldb::DescriptionLevel description_level) c
               m_watch_write ? "w" : "");
 
     if (description_level >= lldb::eDescriptionLevelFull) {
-        if (m_decl_str.c_str())
+        if (!m_decl_str.empty())
             s->Printf("\n    declare @ '%s'", m_decl_str.c_str());
         if (GetConditionText())
             s->Printf("\n    condition = '%s'", GetConditionText());
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index 4756e930ecb..a68bbd3e834 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -16,10 +16,14 @@
 #include "lldb/Breakpoint/Watchpoint.h"
 #include "lldb/Breakpoint/WatchpointList.h"
 #include "lldb/Core/StreamString.h"
+#include "lldb/Core/ValueObject.h"
+#include "lldb/Core/ValueObjectVariable.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
-#include "lldb/Target/Target.h"
 #include "lldb/Interpreter/CommandCompletions.h"
+#include "lldb/Symbol/Variable.h"
+#include "lldb/Symbol/VariableList.h"
+#include "lldb/Target/Target.h"
 
 #include <vector>
 
@@ -157,6 +161,7 @@ CommandObjectMultiwordWatchpoint::CommandObjectMultiwordWatchpoint(CommandInterp
     CommandObjectSP delete_command_object (new CommandObjectWatchpointDelete (interpreter));
     CommandObjectSP ignore_command_object (new CommandObjectWatchpointIgnore (interpreter));
     CommandObjectSP modify_command_object (new CommandObjectWatchpointModify (interpreter));
+    CommandObjectSP set_command_object (new CommandObjectWatchpointSet (interpreter));
 
     list_command_object->SetCommandName ("watchpoint list");
     enable_command_object->SetCommandName("watchpoint enable");
@@ -164,6 +169,7 @@ CommandObjectMultiwordWatchpoint::CommandObjectMultiwordWatchpoint(CommandInterp
     delete_command_object->SetCommandName("watchpoint delete");
     ignore_command_object->SetCommandName("watchpoint ignore");
     modify_command_object->SetCommandName("watchpoint modify");
+    set_command_object->SetCommandName("watchpoint set");
 
     status = LoadSubCommand ("list",       list_command_object);
     status = LoadSubCommand ("enable",     enable_command_object);
@@ -171,6 +177,7 @@ CommandObjectMultiwordWatchpoint::CommandObjectMultiwordWatchpoint(CommandInterp
     status = LoadSubCommand ("delete",     delete_command_object);
     status = LoadSubCommand ("ignore",     ignore_command_object);
     status = LoadSubCommand ("modify",     modify_command_object);
+    status = LoadSubCommand ("set",        set_command_object);
 }
 
 CommandObjectMultiwordWatchpoint::~CommandObjectMultiwordWatchpoint()
@@ -844,3 +851,202 @@ CommandObjectWatchpointModify::Execute
 
     return result.Succeeded();
 }
+
+//-------------------------------------------------------------------------
+// CommandObjectWatchpointSet
+//-------------------------------------------------------------------------
+#pragma mark Set
+
+CommandObjectWatchpointSet::CommandObjectWatchpointSet (CommandInterpreter &interpreter) :
+    CommandObject (interpreter,
+                   "watchpoint set",
+                   "Set a watchpoint. "
+                   "You can choose to watch a variable in scope with just the '-w' option. "
+                   "If you use the '-x' option to specify the byte size, it is implied "
+                   "that the remaining string is evaluated as an expression with the result "
+                   "interpreted as an address to watch for, i.e., the pointee is watched. "
+                   "If no '-w' option is specified, it defaults to read_write. "
+                   "Note that hardware resources for watching are often limited.",
+                   NULL,
+                   eFlagProcessMustBeLaunched | eFlagProcessMustBePaused),
+    m_option_group (interpreter),
+    m_option_watchpoint()
+{
+    SetHelpLong(
+"Examples: \n\
+\n\
+    watchpoint set -w read_wriate my_global_var \n\
+    # Watch my_global_var for read/write access.\n\
+\n\
+    watchpoint set -w write -x 1 foo + 32\n\
+    # Watch write access for the 1-byte region pointed to by the address 'foo + 32'.\n");
+
+    CommandArgumentEntry arg;
+    CommandArgumentData var_name_arg, expression_arg;
+        
+    // Define the first variant of this arg.
+    var_name_arg.arg_type = eArgTypeVarName;
+    var_name_arg.arg_repetition = eArgRepeatPlain;
+
+    // Define the second variant of this arg.
+    expression_arg.arg_type = eArgTypeExpression;
+    expression_arg.arg_repetition = eArgRepeatPlain;
+        
+    // Push the two variants into the argument entry.
+    arg.push_back (var_name_arg);
+    arg.push_back (expression_arg);
+        
+    // Push the data for the first argument into the m_arguments vector.
+    m_arguments.push_back (arg);
+        
+    m_option_group.Append (&m_option_watchpoint, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
+    m_option_group.Finalize();
+}
+
+CommandObjectWatchpointSet::~CommandObjectWatchpointSet ()
+{
+}
+
+Options *
+CommandObjectWatchpointSet::GetOptions ()
+{
+    return &m_option_group;
+}
+
+bool
+CommandObjectWatchpointSet::Execute
+(
+    Args& command,
+    CommandReturnObject &result
+)
+{
+    Target *target = m_interpreter.GetDebugger().GetSelectedTarget().get();
+    ExecutionContext exe_ctx(m_interpreter.GetExecutionContext());
+    StackFrame *frame = exe_ctx.GetFramePtr();
+    if (frame == NULL)
+    {
+        result.AppendError ("you must be stopped in a valid stack frame to set a watchpoint.");
+        result.SetStatus (eReturnStatusFailed);
+        return false;
+    }
+
+    // Be careful about the stack frame, if any summary formatter runs code, it might clear the StackFrameList
+    // for the thread.  So hold onto a shared pointer to the frame so it stays alive.
+    bool get_file_globals = true;
+    VariableList *variable_list = frame->GetVariableList (get_file_globals);
+
+    bool watch_address = (m_option_watchpoint.watch_size > 0);
+        
+    // If no '-w' is specified, default to '-w read_write'.
+    if (!m_option_watchpoint.watch_variable)
+    {
+        m_option_watchpoint.watch_variable = true;
+        m_option_watchpoint.watch_type = OptionGroupWatchpoint::eWatchReadWrite;
+    }
+    // It's possible to specify an address to watch for with the '-x' option.
+    if (!variable_list && !watch_address)
+    {
+        result.GetErrorStream().Printf("error: no variables found, did you forget to use '-x' option to watch an address?\n");
+        result.SetStatus(eReturnStatusFailed);
+        return false;
+    }
+    // If thre's no argument, it is an error.
+    if (command.GetArgumentCount() <= 0) {
+        result.GetErrorStream().Printf("error: specify your target variable (no '-x') or expression (with '-x') to watch for\n");
+        result.SetStatus(eReturnStatusFailed);
+        return false;
+    }        
+
+    // We passed the sanity check for the options.
+    // Proceed to set the watchpoint now.
+    lldb::addr_t addr = 0;
+    size_t size = 0;
+
+    VariableSP var_sp;
+    ValueObjectSP valobj_sp;
+    Stream &output_stream = result.GetOutputStream();
+
+    if (watch_address) {
+        std::string expr_str;
+        command.GetQuotedCommandString(expr_str);
+        const bool coerce_to_id = true;
+        const bool unwind_on_error = true;
+        const bool keep_in_memory = false;
+        ExecutionResults expr_result = target->EvaluateExpression (expr_str.c_str(), 
+                                                                   frame, 
+                                                                   eExecutionPolicyOnlyWhenNeeded,
+                                                                   coerce_to_id,
+                                                                   unwind_on_error, 
+                                                                   keep_in_memory, 
+                                                                   eNoDynamicValues, 
+                                                                   valobj_sp);
+        if (expr_result != eExecutionCompleted) {
+            result.GetErrorStream().Printf("error: expression evaluation of address to watch failed\n");
+            result.SetStatus(eReturnStatusFailed);
+        }
+
+        // Get the address to watch.
+        addr = valobj_sp->GetValueAsUnsigned(0);
+        if (!addr) {
+            result.GetErrorStream().Printf("error: expression did not evaluate to an address\n");
+            result.SetStatus(eReturnStatusFailed);
+            return false;
+        }
+        size = m_option_watchpoint.watch_size;
+    } else {
+        // A simple watch variable gesture allows only one argument.
+        if (m_option_watchpoint.watch_size == 0 && command.GetArgumentCount() != 1) {
+            result.GetErrorStream().Printf("error: specify exactly one variable with the '-w' option, i.e., no '-x'\n");
+            result.SetStatus(eReturnStatusFailed);
+            return false;
+        }
+
+        // Things have checked out ok...
+        Error error;
+        uint32_t expr_path_options = StackFrame::eExpressionPathOptionCheckPtrVsMember;
+        valobj_sp = frame->GetValueForVariableExpressionPath (command.GetArgumentAtIndex(0), 
+                                                              eNoDynamicValues, 
+                                                              expr_path_options,
+                                                              var_sp,
+                                                              error);
+        if (valobj_sp) {
+            AddressType addr_type;
+            addr = valobj_sp->GetAddressOf(false, &addr_type);
+            if (addr_type == eAddressTypeLoad) {
+                // We're in business.
+                // Find out the size of this variable.
+                size = valobj_sp->GetByteSize();
+            }
+        } else {
+            const char *error_cstr = error.AsCString(NULL);
+            if (error_cstr)
+                result.GetErrorStream().Printf("error: %s\n", error_cstr);
+            else
+                result.GetErrorStream().Printf ("error: unable to find any variable expression path that matches '%s'\n",
+                                                command.GetArgumentAtIndex(0));
+            return false;
+        }
+    }
+
+    // Now it's time to create the watchpoint.
+    uint32_t watch_type = m_option_watchpoint.watch_type;
+    Watchpoint *wp = exe_ctx.GetTargetRef().CreateWatchpoint(addr, size, watch_type).get();
+    if (wp) {
+        if (var_sp && var_sp->GetDeclaration().GetFile()) {
+            StreamString ss;
+            // True to show fullpath for declaration file.
+            var_sp->GetDeclaration().DumpStopContext(&ss, true);
+            wp->SetDeclInfo(ss.GetString());
+        }
+        StreamString ss;
+        output_stream.Printf("Watchpoint created: ");
+        wp->GetDescription(&output_stream, lldb::eDescriptionLevelFull);
+        output_stream.EOL();
+        result.SetStatus(eReturnStatusSuccessFinishResult);
+    } else {
+        result.AppendErrorWithFormat("Watchpoint creation failed.\n");
+        result.SetStatus(eReturnStatusFailed);
+    }
+
+    return result.Succeeded();
+}
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.h b/lldb/source/Commands/CommandObjectWatchpoint.h
index 399aa1a775c..c61ebf44d3a 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.h
+++ b/lldb/source/Commands/CommandObjectWatchpoint.h
@@ -17,6 +17,7 @@
 // Project includes
 #include "lldb/Interpreter/CommandObjectMultiword.h"
 #include "lldb/Interpreter/Options.h"
+#include "lldb/Interpreter/OptionGroupWatchpoint.h"
 
 namespace lldb_private {
 
@@ -242,6 +243,31 @@ private:
     CommandOptions m_options;
 };
 
+//-------------------------------------------------------------------------
+// CommandObjectWatchpointSet
+//-------------------------------------------------------------------------
+
+class CommandObjectWatchpointSet : public CommandObject
+{
+public:
+
+    CommandObjectWatchpointSet (CommandInterpreter &interpreter);
+
+    virtual
+    ~CommandObjectWatchpointSet ();
+
+    virtual bool
+    Execute (Args& command,
+             CommandReturnObject &result);
+
+    virtual Options *
+    GetOptions ();
+
+private:
+    OptionGroupOptions m_option_group;
+    OptionGroupWatchpoint m_option_watchpoint;
+};
+
 } // namespace lldb_private
 
 #endif  // liblldb_CommandObjectWatchpoint_h_
diff --git a/lldb/test/functionalities/watchpoint/hello_watchpoint/TestMyFirstWatchpoint.py b/lldb/test/functionalities/watchpoint/hello_watchpoint/TestMyFirstWatchpoint.py
index 5cb8eeb2ca2..30ba801b318 100644
--- a/lldb/test/functionalities/watchpoint/hello_watchpoint/TestMyFirstWatchpoint.py
+++ b/lldb/test/functionalities/watchpoint/hello_watchpoint/TestMyFirstWatchpoint.py
@@ -12,18 +12,30 @@ class HelloWatchpointTestCase(TestBase):
     mydir = os.path.join("functionalities", "watchpoint", "hello_watchpoint")
 
     @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
-    def test_hello_watchpoint_with_dsym(self):
+    def test_hello_watchpoint_with_dsym_using_frame_var(self):
         """Test a simple sequence of watchpoint creation and watchpoint hit."""
         self.buildDsym(dictionary=self.d)
         self.setTearDownCleanup(dictionary=self.d)
         self.hello_watchpoint()
 
-    def test_hello_watchpoint_with_dwarf(self):
+    def test_hello_watchpoint_with_dwarf_using_frame_var(self):
         """Test a simple sequence of watchpoint creation and watchpoint hit."""
         self.buildDwarf(dictionary=self.d)
         self.setTearDownCleanup(dictionary=self.d)
         self.hello_watchpoint()
 
+    def test_hello_watchpoint_with_dsym_using_watchpoint_set(self):
+        """Test a simple sequence of watchpoint creation and watchpoint hit."""
+        self.buildDsym(dictionary=self.d)
+        self.setTearDownCleanup(dictionary=self.d)
+        self.hello_watchpoint(use_frame_var=False)
+
+    def test_hello_watchpoint_with_dwarf_using_watchpoint_set(self):
+        """Test a simple sequence of watchpoint creation and watchpoint hit."""
+        self.buildDwarf(dictionary=self.d)
+        self.setTearDownCleanup(dictionary=self.d)
+        self.hello_watchpoint(use_frame_var=False)
+
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -37,7 +49,7 @@ class HelloWatchpointTestCase(TestBase):
         self.exe_name = self.testMethodName
         self.d = {'C_SOURCES': self.source, 'EXE': self.exe_name}
 
-    def hello_watchpoint(self):
+    def hello_watchpoint(self, use_frame_var=True):
         """Test a simple sequence of watchpoint creation and watchpoint hit."""
         exe = os.path.join(os.getcwd(), self.exe_name)
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
@@ -58,9 +70,14 @@ class HelloWatchpointTestCase(TestBase):
 
         # Now let's set a write-type watchpoint for 'global'.
         # There should be only one watchpoint hit (see main.c).
-        self.expect("frame variable -w write -g -L global", WATCHPOINT_CREATED,
-            substrs = ['Watchpoint created', 'size = 4', 'type = w',
-                       '%s:%d' % (self.source, self.decl)])
+        if use_frame_var:
+            self.expect("frame variable -w write -g -L global", WATCHPOINT_CREATED,
+                substrs = ['Watchpoint created', 'size = 4', 'type = w',
+                           '%s:%d' % (self.source, self.decl)])
+        else:
+            self.expect("watchpoint set -w write global", WATCHPOINT_CREATED,
+                substrs = ['Watchpoint created', 'size = 4', 'type = w',
+                           '%s:%d' % (self.source, self.decl)])
 
         # Use the '-v' option to do verbose listing of the watchpoint.
         # The hit count should be 0 initially.
diff --git a/lldb/test/functionalities/watchpoint/watchpoint_set_command/Makefile b/lldb/test/functionalities/watchpoint/watchpoint_set_command/Makefile
new file mode 100644
index 00000000000..314f1cb2f07
--- /dev/null
+++ b/lldb/test/functionalities/watchpoint/watchpoint_set_command/Makefile
@@ -0,0 +1,5 @@
+LEVEL = ../../../make
+
+CXX_SOURCES := main.cpp
+
+include $(LEVEL)/Makefile.rules
diff --git a/lldb/test/functionalities/watchpoint/watchpoint_set_command/TestWatchLocationWithWatchSet.py b/lldb/test/functionalities/watchpoint/watchpoint_set_command/TestWatchLocationWithWatchSet.py
new file mode 100644
index 00000000000..56e6e502023
--- /dev/null
+++ b/lldb/test/functionalities/watchpoint/watchpoint_set_command/TestWatchLocationWithWatchSet.py
@@ -0,0 +1,102 @@
+"""
+Test lldb watchpoint that uses '-x size' to watch a pointed location with size.
+"""
+
+import os, time
+import unittest2
+import lldb
+from lldbtest import *
+
+class WatchLocationUsingWatchpointSetTestCase(TestBase):
+
+    mydir = os.path.join("functionalities", "watchpoint", "watchpoint_set_command")
+
+    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
+    def test_watchlocation_with_dsym_using_watchpoint_set(self):
+        """Test watching a location with 'watchpoint set -w write -x size' option."""
+        self.buildDsym(dictionary=self.d)
+        self.setTearDownCleanup(dictionary=self.d)
+        self.watchlocation_using_watchpoint_set()
+
+    def test_watchlocation_with_dwarf_using_watchpoint_set(self):
+        """Test watching a location with 'watchpoint set -w write -x size' option."""
+        self.buildDwarf(dictionary=self.d)
+        self.setTearDownCleanup(dictionary=self.d)
+        self.watchlocation_using_watchpoint_set()
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Our simple source filename.
+        self.source = 'main.cpp'
+        # Find the line number to break inside main().
+        self.line = line_number(self.source, '// Set break point at this line.')
+        # This is for verifying that watch location works.
+        self.violating_func = "do_bad_thing_with_location";
+        # Build dictionary to have unique executable names for each test method.
+        self.exe_name = self.testMethodName
+        self.d = {'CXX_SOURCES': self.source, 'EXE': self.exe_name}
+
+    def watchlocation_using_watchpoint_set(self):
+        """Test watching a location with '-x size' option."""
+        exe = os.path.join(os.getcwd(), self.exe_name)
+        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
+
+        # Add a breakpoint to set a watchpoint when stopped on the breakpoint.
+        self.expect("breakpoint set -l %d" % self.line, BREAKPOINT_CREATED,
+            startstr = "Breakpoint created: 1: file ='%s', line = %d, locations = 1" %
+                       (self.source, self.line))
+
+        # Run the program.
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        # We should be stopped again due to the breakpoint.
+        # The stop reason of the thread should be breakpoint.
+        self.expect("thread list", STOPPED_DUE_TO_BREAKPOINT,
+            substrs = ['stopped',
+                       'stop reason = breakpoint'])
+
+        # Now let's set a write-type watchpoint pointed to by 'g_char_ptr' and
+        # with offset as 7.
+        # The main.cpp, by design, misbehaves by not following the agreed upon
+        # protocol of only accessing the allowable index range of [0, 6].
+        self.expect("watchpoint set -w write -x 1 g_char_ptr + 7", WATCHPOINT_CREATED,
+            substrs = ['Watchpoint created', 'size = 1', 'type = w'])
+        self.runCmd("expr unsigned val = *g_char_ptr; val")
+        self.expect(self.res.GetOutput().splitlines()[0], exe=False,
+            endstr = ' = 0')
+
+        # Use the '-v' option to do verbose listing of the watchpoint.
+        # The hit count should be 0 initially.
+        self.expect("watchpoint list -v",
+            substrs = ['hit_count = 0'])
+
+        self.runCmd("process continue")
+
+        # We should be stopped again due to the watchpoint (write type), but
+        # only once.  The stop reason of the thread should be watchpoint.
+        self.expect("thread list", STOPPED_DUE_TO_WATCHPOINT,
+            substrs = ['stopped',
+                       'stop reason = watchpoint',
+                       self.violating_func])
+
+        # Switch to the thread stopped due to watchpoint and issue some commands.
+        self.switch_to_thread_with_stop_reason(lldb.eStopReasonWatchpoint)
+        self.runCmd("thread backtrace")
+        self.runCmd("expr unsigned val = g_char_ptr[7]; val")
+        self.expect(self.res.GetOutput().splitlines()[0], exe=False,
+            endstr = ' = 99')
+
+        # Use the '-v' option to do verbose listing of the watchpoint.
+        # The hit count should now be 1.
+        self.expect("watchpoint list -v",
+            substrs = ['hit_count = 1'])
+
+        self.runCmd("thread backtrace all")
+
+
+if __name__ == '__main__':
+    import atexit
+    lldb.SBDebugger.Initialize()
+    atexit.register(lambda: lldb.SBDebugger.Terminate())
+    unittest2.main()
diff --git a/lldb/test/functionalities/watchpoint/watchpoint_set_command/main.cpp b/lldb/test/functionalities/watchpoint/watchpoint_set_command/main.cpp
new file mode 100644
index 00000000000..83ea085ce36
--- /dev/null
+++ b/lldb/test/functionalities/watchpoint/watchpoint_set_command/main.cpp
@@ -0,0 +1,109 @@
+//===-- main.cpp ------------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// C includes
+#include <pthread.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+pthread_t g_thread_1 = NULL;
+pthread_t g_thread_2 = NULL;
+pthread_t g_thread_3 = NULL;
+
+char *g_char_ptr = NULL;
+
+void
+do_bad_thing_with_location(unsigned index, char *char_ptr, char new_val)
+{
+    unsigned what = new_val;
+    printf("new value written to array(%p) and index(%u) = %u\n", char_ptr, index, what);
+    char_ptr[index] = new_val;
+}
+
+uint32_t access_pool (uint32_t flag = 0);
+
+uint32_t
+access_pool (uint32_t flag)
+{
+    static pthread_mutex_t g_access_mutex = PTHREAD_MUTEX_INITIALIZER;
+    static unsigned idx = 0; // Well-behaving thread only writes into indexs from 0..6.
+    if (flag == 0)
+        ::pthread_mutex_lock (&g_access_mutex);
+
+    // idx valid range is [0, 6].
+    if (idx > 6)
+        idx = 0;
+
+    if (flag != 0) {
+        // Write into a forbidden area.
+        do_bad_thing_with_location(7, g_char_ptr, 99);
+    }
+
+    unsigned index = idx++;
+
+    if (flag == 0)
+        ::pthread_mutex_unlock (&g_access_mutex);
+    return g_char_ptr[index];
+}
+
+void *
+thread_func (void *arg)
+{
+    uint32_t thread_index = *((uint32_t *)arg);
+    printf ("%s (thread index = %u) startng...\n", __FUNCTION__, thread_index);
+
+    uint32_t count = 0;
+    uint32_t val;
+    while (count++ < 15)
+    {
+        // random micro second sleep from zero to 3 seconds
+        int usec = ::rand() % 3000000;
+        printf ("%s (thread = %u) doing a usleep (%d)...\n", __FUNCTION__, thread_index, usec);
+        ::usleep (usec);
+        
+        if (count < 7)
+            val = access_pool ();
+        else
+            val = access_pool (1);
+                
+        printf ("%s (thread = %u) after usleep access_pool returns %d (count=%d)...\n", __FUNCTION__, thread_index, val, count);
+    }
+    printf ("%s (thread index = %u) exiting...\n", __FUNCTION__, thread_index);
+    return NULL;
+}
+
+
+int main (int argc, char const *argv[])
+{
+    int err;
+    void *thread_result = NULL;
+    uint32_t thread_index_1 = 1;
+    uint32_t thread_index_2 = 2;
+    uint32_t thread_index_3 = 3;
+
+    g_char_ptr = (char *)malloc (10);
+    for (int i = 0; i < 10; ++i)
+        *g_char_ptr = 0;
+
+    // Create 3 threads
+    err = ::pthread_create (&g_thread_1, NULL, thread_func, &thread_index_1);
+    err = ::pthread_create (&g_thread_2, NULL, thread_func, &thread_index_2);
+    err = ::pthread_create (&g_thread_3, NULL, thread_func, &thread_index_3);
+
+    printf ("Before turning all three threads loose...\n"); // Set break point at this line.
+
+    // Join all of our threads
+    err = ::pthread_join (g_thread_1, &thread_result);
+    err = ::pthread_join (g_thread_2, &thread_result);
+    err = ::pthread_join (g_thread_3, &thread_result);
+
+    return 0;
+}