Fix hanging MDIA testcases.

Change-Id: I3cec520ccf3fcbeb61f8fdaed7d22b715154e849
RTC: 50861
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/2014
Tested-by: Jenkins Server
Reviewed-by: Christopher T. Phan <cphan@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

2 files changed, 130 insertions, 109 deletions

diff --git a/src/usr/diag/mdia/mdiasm.C b/src/usr/diag/mdia/mdiasm.C
index a1835d525..68811dad4 100644
--- a/src/usr/diag/mdia/mdiasm.C
+++ b/src/usr/diag/mdia/mdiasm.C
@@ -49,7 +49,7 @@ void StateMachine::running(bool & o_running)
 {
     mutex_lock(&iv_mutex);
 
-    o_running = !iv_done;
+    o_running = !(iv_done || iv_shutdown);
 
     mutex_unlock(&iv_mutex);
 }
@@ -61,27 +61,30 @@ void StateMachine::processCommandTimeout(const MonitorIDs & i_monitorIDs)
 
     mutex_lock(&iv_mutex);
 
-    for(MonitorIDs::const_iterator monitorIt = i_monitorIDs.begin();
-        monitorIt != i_monitorIDs.end();
-        ++monitorIt)
+    if(!iv_shutdown)
     {
-        for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
-            wit != iv_workFlowProperties.end();
-            ++wit)
+        for(MonitorIDs::const_iterator monitorIt = i_monitorIDs.begin();
+                monitorIt != i_monitorIDs.end();
+                ++monitorIt)
         {
-            if((*wit)->timer == *monitorIt)
+            for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
+                    wit != iv_workFlowProperties.end();
+                    ++wit)
             {
-                (*wit)->status = COMMAND_TIMED_OUT;
-                wkflprop = *wit;
-                break;
+                if((*wit)->timer == *monitorIt)
+                {
+                    (*wit)->status = COMMAND_TIMED_OUT;
+                    wkflprop = *wit;
+                    break;
+                }
             }
         }
-    }
 
-    //Satisfies last/one target remaining to run maint cmds.
-    //If no match found, implies SM has already processed event(s).
-    if(wkflprop)
-        scheduleWorkItem(*wkflprop);
+        //Satisfies last/one target remaining to run maint cmds.
+        //If no match found, implies SM has already processed event(s).
+        if(wkflprop)
+            scheduleWorkItem(*wkflprop);
+    }
 
     mutex_unlock(&iv_mutex);
 }
@@ -174,7 +177,7 @@ void StateMachine::wait()
 
     // wait for everything to finish
 
-    while(!iv_done)
+    while(!iv_done && !iv_shutdown)
     {
         sync_cond_wait(&iv_cond, &iv_mutex);
     }
@@ -188,6 +191,8 @@ void StateMachine::start()
 
     MDIA_FAST("sm: starting up");
 
+    iv_shutdown = false;
+
     // schedule the first work items for all target / workFlow associations
 
     for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
@@ -296,65 +301,67 @@ bool StateMachine::executeWorkItem(WorkFlowProperties * i_wfp)
 
     // ensure this thread sees the most recent state
 
-    bool async = workItemIsAsync(*i_wfp);
+    if(!iv_shutdown)
+    {
+        bool async = workItemIsAsync(*i_wfp);
 
-    uint64_t workItem = *i_wfp->workItem;
+        uint64_t workItem = *i_wfp->workItem;
 
-    MDIA_FAST("sm: executing work item %d for: %p",
-            workItem, getTarget(*i_wfp));
+        MDIA_FAST("sm: executing work item %d for: %p",
+                workItem, getTarget(*i_wfp));
 
-    mutex_unlock(&iv_mutex);
+        mutex_unlock(&iv_mutex);
 
-    errlHndl_t err = 0;
+        errlHndl_t err = 0;
 
-    switch(workItem)
-    {
-        // do the appropriate thing based on the phase for this target
+        switch(workItem)
+        {
+            // do the appropriate thing based on the phase for this target
 
-        case START_PATTERN_0:
-        case START_PATTERN_1:
-        case START_PATTERN_2:
-        case START_PATTERN_3:
-        case START_PATTERN_4:
-        case START_PATTERN_5:
-        case START_PATTERN_6:
-        case START_PATTERN_7:
-        case START_RANDOM_PATTERN:
-        case START_SCRUB:
+            case START_PATTERN_0:
+            case START_PATTERN_1:
+            case START_PATTERN_2:
+            case START_PATTERN_3:
+            case START_PATTERN_4:
+            case START_PATTERN_5:
+            case START_PATTERN_6:
+            case START_PATTERN_7:
+            case START_RANDOM_PATTERN:
+            case START_SCRUB:
 
-            err = doMaintCommand(*i_wfp);
+                err = doMaintCommand(*i_wfp);
 
-            break;
+                break;
 
-        default:
-            break;
-    }
+            default:
+                break;
+        }
 
-    mutex_lock(&iv_mutex);
+        mutex_lock(&iv_mutex);
 
-    if(err)
-    {
-        // stop the workFlow for this target
+        if(err)
+        {
+            // stop the workFlow for this target
 
-        i_wfp->status = FAILED;
-        i_wfp->log = err;
-    }
+            i_wfp->status = FAILED;
+            i_wfp->log = err;
+        }
 
-    else if(!async)
-    {
-        // sync work item -
-        // move the workFlow pointer to the next phase
-        ++i_wfp->workItem;
-    }
+        else if(!async)
+        {
+            // sync work item -
+            // move the workFlow pointer to the next phase
+            ++i_wfp->workItem;
+        }
 
-    if(err || !async)
-    {
-        // check to see if this was the last workFlow
-        // in progress (if there was an error), or for sync
-        // work items, schedule the next work item
-        dispatched = scheduleWorkItem(*i_wfp);
+        if(err || !async)
+        {
+            // check to see if this was the last workFlow
+            // in progress (if there was an error), or for sync
+            // work items, schedule the next work item
+            dispatched = scheduleWorkItem(*i_wfp);
+        }
     }
-
     mutex_unlock(&iv_mutex);
 
     return dispatched;
@@ -374,16 +381,18 @@ errlHndl_t StateMachine::doMaintCommand(WorkFlowProperties & i_wfp)
     ecmdDataBufferBase startAddr(64), endAddr(64);
     mss_MaintCmd * cmd = NULL;
 
+    mutex_lock(&iv_mutex);
+
     // starting a maint cmd ...  register a timeout monitor
     uint64_t monitorId = getMonitor().addMonitor(timeout);
 
-    mutex_lock(&iv_mutex);
-
     i_wfp.timer = monitorId;
     workItem = *i_wfp.workItem;
     restart = i_wfp.restartCommand;
     targetMba = getTarget(i_wfp);
 
+    mutex_unlock(&iv_mutex);
+
     fapi::Target fapiMba(TARGET_TYPE_MBA_CHIPLET, targetMba);
 
     do {
@@ -478,6 +487,8 @@ errlHndl_t StateMachine::doMaintCommand(WorkFlowProperties & i_wfp)
 
     } while(0);
 
+    mutex_lock(&iv_mutex);
+
     if(err)
     {
         MDIA_FAST("sm: Running Maint Cmd failed");
@@ -514,6 +525,8 @@ CommandMonitor & StateMachine::getMonitor()
 
 bool StateMachine::processMaintCommandEvent(const MaintCommandEvent & i_event)
 {
+    bool resume = true, dispatched = false;
+
     mutex_lock(&iv_mutex);
 
     WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
@@ -529,64 +542,62 @@ bool StateMachine::processMaintCommandEvent(const MaintCommandEvent & i_event)
     if(wit == iv_workFlowProperties.end())
     {
         MDIA_ERR("sm: did not find target");
-
-        return false;
     }
+    else if(!iv_shutdown)
+    {
+        WorkFlowProperties & wfp = **wit;
 
-    bool resume = true, dispatched = false;
+        // always unregister any existing maint cmd monitor
 
-    WorkFlowProperties & wfp = **wit;
+        getMonitor().removeMonitor(wfp.timer);
 
-    // always unregister any existing maint cmd monitor
+        MDIA_FAST("sm: processing event for: %p", getTarget(wfp));
 
-    getMonitor().removeMonitor(wfp.timer);
+        switch(i_event.type)
+        {
+            case COMMAND_COMPLETE:
 
-    MDIA_FAST("sm: processing event for: %p", getTarget(wfp));
+                // command stopped or complete at end of last rank
 
-    switch(i_event.type)
-    {
-        case COMMAND_COMPLETE:
+                wfp.restartCommand = false;
 
-            // command stopped or complete at end of last rank
+                // move to the next command
 
-            wfp.restartCommand = false;
+                ++wfp.workItem;
 
-            // move to the next command
+                break;
 
-            ++wfp.workItem;
+            case COMMAND_STOPPED:
 
-            break;
+                // command stopped at end of some other rank
 
-        case COMMAND_STOPPED:
+                wfp.restartCommand = true;
 
-            // command stopped at end of some other rank
+                break;
 
-            wfp.restartCommand = true;
+            case SKIP_MBA:
 
-            break;
+                // stop testing on this mba
 
-        case SKIP_MBA:
+                wfp.status = COMPLETE;
 
-            // stop testing on this mba
+                break;
 
-            wfp.status = COMPLETE;
+            case RESET_TIMER:
 
-            break;
+                // fall through
+            default:
 
-        case RESET_TIMER:
+                resume = false;
+                break;
+        }
 
-            // fall through
-        default:
+        // schedule the next work item
 
-            resume = false;
-            break;
+        if(resume)
+            dispatched = scheduleWorkItem(wfp);
     }
 
-    // schedule the next work item
-
-    if(resume)
-        dispatched = scheduleWorkItem(wfp);
-
     mutex_unlock(&iv_mutex);
 
     return dispatched;
@@ -637,28 +648,32 @@ void StateMachine::shutdown()
 {
     mutex_lock(&iv_mutex);
 
+    Util::ThreadPool<WorkItem> * tp = iv_tp;
+    CommandMonitor * monitor = iv_monitor;
+
+    iv_tp = 0;
+    iv_monitor = 0;
+
+    iv_shutdown = true;
+
+    mutex_unlock(&iv_mutex);
+
     MDIA_FAST("sm: shutting down...");
 
-    if(iv_tp)
+    if(tp)
     {
         MDIA_FAST("Stopping threadPool...");
-        iv_tp->shutdown();
-        delete iv_tp;
-        iv_tp = 0;
+        tp->shutdown();
+        delete tp;
     }
 
-    if(iv_monitor)
+    if(monitor)
     {
         MDIA_FAST("Stopping monitor...");
-        iv_monitor->shutdown();
-        delete iv_monitor;
-        iv_monitor = 0;
+        monitor->shutdown();
+        delete monitor;
     }
 
-    mutex_unlock(&iv_mutex);
-
-    reset();
-
     MDIA_FAST("sm: ...shutdown complete");
 }
 
@@ -670,7 +685,8 @@ StateMachine::~StateMachine()
     mutex_destroy(&iv_mutex);
 }
 
-StateMachine::StateMachine() : iv_monitor(0), iv_done(true), iv_tp(0)
+StateMachine::StateMachine() : iv_monitor(0), iv_done(true), iv_shutdown(false),
+    iv_tp(0)
 {
     mutex_init(&iv_mutex);
     sync_cond_init(&iv_cond);
diff --git a/src/usr/diag/mdia/mdiasm.H b/src/usr/diag/mdia/mdiasm.H
index 7ac8e22a5..5957366a3 100644
--- a/src/usr/diag/mdia/mdiasm.H
+++ b/src/usr/diag/mdia/mdiasm.H
@@ -231,6 +231,11 @@ class StateMachine
         bool iv_done;
 
         /**
+         * @brief shutdown indicator
+         */
+        bool iv_shutdown;
+
+        /**
          * @brief thread pool
          */
         Util::ThreadPool<WorkItem> * iv_tp;