Process Maintenance Command timeout

When a maintenance cmd timesout, set the target/s WorkFlowStatus
to the appropriate state such that the state machine can
take further action.

RTC: 41894
Change-Id: Iaf5b2b5ab07f9269379b011b720df760e92e178f
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1200
Tested-by: Jenkins Server
Reviewed-by: Zane Shelley <zshelle@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

11 files changed, 501 insertions, 95 deletions

diff --git a/src/usr/diag/mdia/mdiafwd.H b/src/usr/diag/mdia/mdiafwd.H
index dcc571994..4c090a3bb 100644
--- a/src/usr/diag/mdia/mdiafwd.H
+++ b/src/usr/diag/mdia/mdiafwd.H
@@ -129,6 +129,11 @@ typedef std::map<TARGETING::TargetHandle_t, WorkFlow> WorkFlowAssocMap;
 typedef WorkFlowAssocMap::const_iterator WorkFlowAssoc;
 
 /**
+* @brief MonitorIDs container of monitorIDs unique to a target
+*/
+typedef std::vector<uint64_t> MonitorIDs;
+
+/**
  * @brief getMbaDiagnosticMode get the mode (scrub, init, one, four, nine)
  *
  * @param[in] i_globals policy flags needed to determine the mode
diff --git a/src/usr/diag/mdia/mdiamonitor.C b/src/usr/diag/mdia/mdiamonitor.C
index 72c7002e7..3d0a15fdb 100755
--- a/src/usr/diag/mdia/mdiamonitor.C
+++ b/src/usr/diag/mdia/mdiamonitor.C
@@ -110,7 +110,7 @@ void CommandMonitor::threadMain(StateMachine & i_sm)
     }
 
     bool shutdown = false;
-    std::vector<uint64_t> monitorsTimedout;
+    MonitorIDs monitorsTimedout;
 
     while(true)
     {
@@ -266,8 +266,8 @@ void CommandMonitor::staticMain(void * i_args)
 }
 
 CommandMonitor::CommandMonitor() :
-    iv_tid(0),
     iv_shutdown(false),
+    iv_tid(0),
     iv_nextMonitor(0)
 {
     mutex_init(&iv_mutex);
diff --git a/src/usr/diag/mdia/mdiamonitor.H b/src/usr/diag/mdia/mdiamonitor.H
index 866829233..7ba686b8f 100755
--- a/src/usr/diag/mdia/mdiamonitor.H
+++ b/src/usr/diag/mdia/mdiamonitor.H
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/diag/mdia/mdiamonitor.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2012
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/diag/mdia/mdiamonitor.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 #ifndef __MSS_MEMDIAG_MDIAMONITOR_H
 #define __MSS_MEMDIAG_MDIAMONITOR_H
 
@@ -91,43 +92,44 @@ class CommandMonitor
         /**
          * @brief dtor
          */
-        ~CommandMonitor();
+        virtual ~CommandMonitor();
 
-    private:
+    protected:
 
         /**
-         * @brief main thread function
+         * @brief serialization mutex
          */
-        static void staticMain(void *);
+        mutex_t iv_mutex;
 
         /**
-         * @brief main thread function
-         *
-         * @param[in] i_sm state machine provided for callback
+         * @brief shutdown request indicator
          */
-        void threadMain(StateMachine & i_sm);
+        bool iv_shutdown;
 
         /**
-         * @brief Get the thread id
-         *
-         * @retval ThreadID of CommandMonitor
+         * @brief aliases. Map containing monitorID key and timeout entries.
          */
-        tid_t getTID();
+        typedef std::map<uint64_t, uint64_t> MonitorMap;
+        typedef MonitorMap::iterator monitorMapIterator;
 
         /**
-         * @brief Is the CommandMonitor shut down
-         *
-         * @retval true if shutdown
-         *         false if not shutdown
+         * @brief maint command monitors
+         */
+        MonitorMap iv_monitors;
+
+    private:
+
+        /**
+         * @brief main thread function
          */
-        bool isShutdown();
+        static void staticMain(void *);
 
         /**
-         * @brief Get the monitor id counter
+         * @brief main thread function
          *
-         * @retval Monitor ID counter
+         * @param[in] i_sm state machine provided for callback
          */
-        uint64_t getMonitorID();
+        virtual void threadMain(StateMachine & i_sm);
 
         /**
          * @brief Get MonitorMap Timeout entry
@@ -139,37 +141,16 @@ class CommandMonitor
         uint64_t getMonitorMapTimeoutEntry(uint64_t i_monitor);
 
         /**
-         * @brief serialization mutex
-         */
-        mutex_t iv_mutex;
-
-        /**
          * @brief thread number
          */
         tid_t iv_tid;
 
         /**
-         * @brief shutdown request indicator
-         */
-        bool iv_shutdown;
-
-        /**
          * @brief monitor ID counter
          */
         uint64_t iv_nextMonitor;
 
         /**
-         * @brief aliases. Map containing monitorID key and timeout entries.
-         */
-        typedef std::map<uint64_t, uint64_t> MonitorMap;
-        typedef MonitorMap::iterator monitorMapIterator;
-
-        /**
-         * @brief maint command monitors
-         */
-        MonitorMap iv_monitors;
-
-        /**
          * @brief list of targets to do polling
          */
         TARGETING::TargetHandleList iv_pollingList;
diff --git a/src/usr/diag/mdia/mdiasm.C b/src/usr/diag/mdia/mdiasm.C
index 9598fbd6c..be5e4a973 100644
--- a/src/usr/diag/mdia/mdiasm.C
+++ b/src/usr/diag/mdia/mdiasm.C
@@ -54,6 +54,38 @@ void StateMachine::running(bool & o_running)
     mutex_unlock(&iv_mutex);
 }
 
+void StateMachine::processCommandTimeout(const MonitorIDs & i_monitorIDs)
+{
+    MDIA_FAST("sm: processCommandTimeout");
+    WorkFlowProperties *wkflprop = NULL;
+
+    mutex_lock(&iv_mutex);
+
+    for(MonitorIDs::const_iterator monitorIt = i_monitorIDs.begin();
+        monitorIt != i_monitorIDs.end();
+        ++monitorIt)
+    {
+        for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
+            wit != iv_workFlowProperties.end();
+            ++wit)
+        {
+            if((*wit)->timer == *monitorIt)
+            {
+                (*wit)->status = COMMAND_TIMED_OUT;
+                wkflprop = *wit;
+                break;
+            }
+        }
+    }
+
+    //Satisfies last/one target remaining to run maint cmds.
+    //If no match found, implies SM has already processed event(s).
+    if(wkflprop)
+        scheduleWorkItem(*wkflprop);
+
+    mutex_unlock(&iv_mutex);
+}
+
 errlHndl_t StateMachine::run(const WorkFlowAssocMap & i_list)
 {
     // load the workflow properties
@@ -72,9 +104,7 @@ errlHndl_t StateMachine::run(const WorkFlowAssocMap & i_list)
 
     mutex_lock(&iv_mutex);
 
-    WorkFlowPropertiesIterator wit;
-
-    for(wit = iv_workFlowProperties.begin();
+    for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
         wit != iv_workFlowProperties.end();
         ++wit)
     {
@@ -98,10 +128,9 @@ void StateMachine::setup(const WorkFlowAssocMap & i_list)
 
     mutex_lock(&iv_mutex);
 
-    WorkFlowAssoc it;
     WorkFlowProperties * p = 0;
 
-    for(it = i_list.begin(); it != i_list.end(); ++it)
+    for(WorkFlowAssoc it = i_list.begin(); it != i_list.end(); ++it)
     {
         // for each target / workFlow assoc,
         // initialize the workFlow progress indicator
@@ -161,9 +190,9 @@ void StateMachine::start()
 
     // schedule the first work items for all target / workFlow associations
 
-    WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
-
-    for(; wit != iv_workFlowProperties.end(); ++wit)
+    for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
+        wit != iv_workFlowProperties.end();
+        ++wit)
     {
         scheduleWorkItem(**wit);
     }
@@ -334,7 +363,7 @@ bool StateMachine::executeWorkItem(WorkFlowProperties * i_wfp)
 
 errlHndl_t StateMachine::doMaintCommand(WorkFlowProperties & i_wfp)
 {
-    errlHndl_t err = 0;
+    errlHndl_t err = NULL;
 
     mutex_lock(&iv_mutex);
 
@@ -556,9 +585,9 @@ bool StateMachine::allWorkFlowsComplete()
 
     bool allWorkFlowsComplete = true;
 
-    WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
-
-    for(; wit != iv_workFlowProperties.end(); ++wit)
+    for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
+        wit != iv_workFlowProperties.end();
+        ++wit)
     {
         if((*wit)->status == IN_PROGRESS)
         {
@@ -574,9 +603,9 @@ void StateMachine::reset()
 {
     mutex_lock(&iv_mutex);
 
-    WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
-
-    for(; wit != iv_workFlowProperties.end(); ++wit)
+    for(WorkFlowPropertiesIterator wit = iv_workFlowProperties.begin();
+        wit != iv_workFlowProperties.end();
+        ++wit)
     {
         if((**wit).log)
         {
@@ -628,7 +657,7 @@ StateMachine::~StateMachine()
     mutex_destroy(&iv_mutex);
 }
 
-StateMachine::StateMachine() : iv_done(true), iv_tp(0), iv_monitor(0)
+StateMachine::StateMachine() : iv_monitor(0), iv_done(true), iv_tp(0)
 {
     mutex_init(&iv_mutex);
     sync_cond_init(&iv_cond);
diff --git a/src/usr/diag/mdia/mdiasm.H b/src/usr/diag/mdia/mdiasm.H
index 60abe14b1..146bb1ce9 100644
--- a/src/usr/diag/mdia/mdiasm.H
+++ b/src/usr/diag/mdia/mdiasm.H
@@ -57,7 +57,7 @@ class StateMachine
          * @param[in] i_monitorIDs Vector of monitorIDs
          */
         virtual void processCommandTimeout(
-                        const std::vector<uint64_t> & i_monitorIDs) {}
+                        const MonitorIDs & i_monitorIDs);
 
         /**
          * @brief run the state machine to completion
@@ -94,13 +94,26 @@ class StateMachine
          */
         virtual ~StateMachine();
 
+    protected:
+
+        /**
+         * @brief serialization mutex
+         */
+        mutex_t iv_mutex;
+
+        /**
+         * @brief command monitor
+         */
+        CommandMonitor * iv_monitor;
+
     private:
 
         /**
          * @brief aliases
          */
         typedef std::vector<WorkFlowProperties *> WorkFlowPropertiesList;
-        typedef std::vector<WorkFlowProperties *>::iterator WorkFlowPropertiesIterator;
+        typedef std::vector<WorkFlowProperties *>::iterator
+                WorkFlowPropertiesIterator;
 
         /**
          * @brief setup load the workflow properties for the provided
@@ -165,7 +178,7 @@ class StateMachine
          *
          * @param[in] i_wfp the target for which a command should be issued
          */
-        errlHndl_t doMaintCommand(WorkFlowProperties & i_wfp);
+        virtual errlHndl_t doMaintCommand(WorkFlowProperties & i_wfp);
 
         /**
          * @brief executeWorkItem work item entry point for
@@ -202,16 +215,6 @@ class StateMachine
         Util::ThreadPool<WorkItem> * iv_tp;
 
         /**
-         * @brief command monitor
-         */
-        CommandMonitor * iv_monitor;
-
-        /**
-         * @brief serialization mutex
-         */
-        mutex_t iv_mutex;
-
-        /**
          * @brief completion condition
          */
         sync_cond_t iv_cond;
diff --git a/src/usr/diag/mdia/test/makefile b/src/usr/diag/mdia/test/makefile
index 237b41868..c24764212 100644
--- a/src/usr/diag/mdia/test/makefile
+++ b/src/usr/diag/mdia/test/makefile
@@ -24,7 +24,7 @@ ROOTPATH = ../../../../..
 
 EXTRAINCDIR += ${ROOTPATH}/src/include/usr/diag
 
-OBJS = mdiafakesm.o
+OBJS = mdiafakesm.o mdiafakecm.o
 
 MODULE = testmdia
 
diff --git a/src/usr/diag/mdia/test/mdiafakecm.C b/src/usr/diag/mdia/test/mdiafakecm.C
new file mode 100644
index 000000000..f049d534b
--- /dev/null
+++ b/src/usr/diag/mdia/test/mdiafakecm.C
@@ -0,0 +1,105 @@
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/diag/mdia/test/mdiafakecm.C $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
+#include "mdiafakecm.H"
+#include "../mdiasm.H"
+#include "../mdiasmimpl.H"
+#include "../mdiafwd.H"
+#include "../mdiatrace.H"
+#include <sys/time.h>
+
+namespace MDIA
+{
+void FakeCommandMonitor::threadMain(StateMachine & i_sm)
+{
+    MDIA_DBG("FakeCommandMonitor: threadMain");
+    static const uint64_t singleStepPauseSecs = 0;
+    static const uint64_t singleStepPauseNSecs = 10000000;
+    static uint64_t wakeupIntervalNanoSecs = 0;
+
+    // periodically wakeup and check for any command
+    // timeouts
+
+    if(TARGETING::is_vpo())
+    {
+        wakeupIntervalNanoSecs = TEN_CTX_SWITCHES_NS;
+    }
+    else
+    {
+        wakeupIntervalNanoSecs = singleStepPauseNSecs;
+    }
+
+    bool shutdown = false;
+    MonitorIDs monitorsTimedout;
+
+    while(true)
+    {
+        if( TARGETING::is_vpo() )
+        {
+            nanosleep(0, wakeupIntervalNanoSecs);
+        }
+        else
+        {
+            nanosleep(singleStepPauseSecs, wakeupIntervalNanoSecs);
+        }
+
+        mutex_lock(&iv_mutex);
+        // check to see if the istep is finished
+        shutdown = iv_shutdown;
+
+        if(!shutdown)
+        {
+            // scan the monitor map and if any
+            // timed out, inform the state machine
+            monitorMapIterator it = iv_monitors.begin();
+
+            while(it != iv_monitors.end())
+            {
+                if(it->second >= wakeupIntervalNanoSecs)
+                {
+                    it->second -= wakeupIntervalNanoSecs;
+                    it++;
+                }
+                else
+                {
+                    monitorsTimedout.push_back(it->first);
+                    //remove the monitor
+                    iv_monitors.erase(it++);
+                }
+            }
+        }
+
+        mutex_unlock(&iv_mutex);
+
+        if(!monitorsTimedout.empty()) {
+            i_sm.processCommandTimeout(monitorsTimedout);
+            monitorsTimedout.clear();
+        }
+
+        if(shutdown)
+        {
+            break;
+        }
+    }
+}
+};
diff --git a/src/usr/diag/mdia/test/mdiafakecm.H b/src/usr/diag/mdia/test/mdiafakecm.H
new file mode 100644
index 000000000..846d93bde
--- /dev/null
+++ b/src/usr/diag/mdia/test/mdiafakecm.H
@@ -0,0 +1,62 @@
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/diag/mdia/test/mdiafakecm.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
+#ifndef __TEST_MDIAFAKECM_H
+#define __TEST_MDIAFAKECM_H
+
+#include "../mdiamonitor.H"
+
+namespace MDIA
+{
+/**
+ * @brief FakeCommandMonitor class that does not support polling.
+ *        Used to test processing a maintenance command timeout.
+ */
+class FakeCommandMonitor : public CommandMonitor
+{
+    public:
+
+        /**
+         * @brief ctor
+         */
+        FakeCommandMonitor(){}
+
+        /**
+         * @brief dtor
+         */
+        ~FakeCommandMonitor(){}
+
+    private:
+
+        /**
+         * @brief threadMain
+         *
+         * main thread function. Does not send fake events to the
+         * StateMachine
+         *
+         * @param[in] i_sm state machine provided for callback
+         */
+        void threadMain(StateMachine & i_sm);
+};
+}
+#endif
diff --git a/src/usr/diag/mdia/test/mdiafakesm.C b/src/usr/diag/mdia/test/mdiafakesm.C
index b53fecced..2186caa86 100644
--- a/src/usr/diag/mdia/test/mdiafakesm.C
+++ b/src/usr/diag/mdia/test/mdiafakesm.C
@@ -22,7 +22,9 @@
  *  IBM_PROLOG_END_TAG
  */
 #include "mdiafakesm.H"
+#include "mdiafakecm.H"
 #include "../mdiatrace.H"
+#include "../mdiasmimpl.H"
 
 namespace MDIA
 {
@@ -149,4 +151,65 @@ bool FakeStateMachine1::isEqual(
 
     return isEqual;
 }
+
+errlHndl_t FakeStateMachine2::doMaintCommand(WorkFlowProperties & i_wfp)
+{
+    MDIA_DBG("FakeStateMachine2: doMaintCommand");
+    mutex_lock(&iv_mutex);
+
+    uint32_t timeout = 100;
+    i_wfp.timer = getMonitor().addMonitor(timeout);
+
+    WorkFlowPhase workItem = *i_wfp.workItem;
+    bool restart = i_wfp.restartCommand;
+
+    mutex_unlock(&iv_mutex);
+
+    switch(workItem) {
+
+        case START_PATTERN_0:
+        case START_PATTERN_1:
+        case START_PATTERN_2:
+        case START_PATTERN_3:
+        case START_PATTERN_4:
+        case START_PATTERN_5:
+        case START_PATTERN_6:
+        case START_PATTERN_7:
+        case START_PATTERN_8:
+        case START_SCRUB:
+
+            mutex_lock(&iv_mutex);
+
+            if(restart) {
+                MDIA_DBG("FakeStateMachine2: issuing increment address on: %p",
+                    getTarget(i_wfp));
+                // TODO...restart the command (increment address)
+            }
+            else {
+                MDIA_DBG("FakeStateMachine2: issuing maint command on: %p",
+                    getTarget(i_wfp));
+                // TODO...start a command
+            }
+
+            mutex_unlock(&iv_mutex);
+
+            break;
+
+        default:
+            break;
+    }
+
+    return 0;
+}
+
+CommandMonitor & FakeStateMachine2::getMonitor()
+{
+    if(!iv_monitor)
+    {
+        MDIA_DBG("FakeStateMachine2: Starting monitor...");
+        iv_monitor = new FakeCommandMonitor();
+        iv_monitor->start(*this);
+    }
+    return *iv_monitor;
+}
 }
diff --git a/src/usr/diag/mdia/test/mdiafakesm.H b/src/usr/diag/mdia/test/mdiafakesm.H
index 61947736a..78e402a38 100644
--- a/src/usr/diag/mdia/test/mdiafakesm.H
+++ b/src/usr/diag/mdia/test/mdiafakesm.H
@@ -25,6 +25,8 @@
 #define __TEST_MDIAFAKESM_H
 
 #include "../mdiasm.H"
+#include "../mdiamonitor.H"
+#include "../mdiafwd.H"
 
 namespace MDIA
 {
@@ -155,5 +157,45 @@ class FakeStateMachine1 : public StateMachine
          */
         mutex_t *iv_mutex;
 };
+
+/**
+ * @brief FakeStateMachine2
+ *
+ * Used to test processing a maint command timeout
+ * from the StateMachine perspective
+ */
+class FakeStateMachine2 : public StateMachine
+{
+    public:
+
+        /**
+         * @brief doMaintCommand maint command work item handler
+         *
+         * determine correct comman and issue it. Does not support
+         * polling/processing fake events.
+         *
+         * @param[in] i_wfp the target for which a command should be issued
+         */
+        errlHndl_t doMaintCommand(WorkFlowProperties & i_wfp);
+
+        /**
+         * @brief ctor
+         */
+        FakeStateMachine2(){}
+
+        /**
+         * @brief dtor
+         */
+        ~FakeStateMachine2(){}
+
+    private:
+
+        /**
+         * @brief getMonitor Create and start the FakeMdiaCMTest
+         *
+         * @retval CommandMonitor reference
+         */
+        CommandMonitor & getMonitor();
+};
 }
 #endif
diff --git a/src/usr/diag/mdia/test/mdiatestsm.H b/src/usr/diag/mdia/test/mdiatestsm.H
index 5d95975c0..725b8a941 100644
--- a/src/usr/diag/mdia/test/mdiatestsm.H
+++ b/src/usr/diag/mdia/test/mdiatestsm.H
@@ -33,6 +33,11 @@
 #include "../mdiasm.H"
 #include "../mdiasmimpl.H"
 #include <diag/mdia/mdiamevent.H>
+#include "../mdiatrace.H"
+#include "../mdiamonitor.H"
+#include "../mdiafwd.H"
+#include <sys/time.h>
+#include "mdiafakesm.H"
 
 using namespace MDIA;
 using namespace TARGETING;
@@ -685,5 +690,116 @@ class MdiaSmTest : public CxxTest::TestSuite
 
             TS_TRACE(EXIT_MRK "testSmRunning");
         }
+
+        void testSmProcessCommandTimeout(void)
+        {
+            using namespace MDIA;
+            using namespace TARGETING;
+
+            TS_TRACE(ENTER_MRK "testSmProcessCommandTimeout");
+
+            FakeStateMachine2 stateMac;
+            WorkFlowAssocMap wkFlEntries;
+
+            TargetHandleList targets;
+            getTargets(3, targets);
+
+            wkFlEntries[targets[0]].push_back(START_SCRUB);
+            wkFlEntries[targets[1]].push_back(START_SCRUB);
+            wkFlEntries[targets[2]].push_back(START_SCRUB);
+            stateMac.setup(wkFlEntries);
+
+            uint64_t indexWkPr = stateMac.iv_workFlowProperties.size();
+            while(indexWkPr-- != 0)
+            {
+                stateMac.doMaintCommand(
+                *stateMac.iv_workFlowProperties[indexWkPr]);
+            }
+
+            static const uint64_t wakeupIntervalNanoSecs = 100000000;
+            nanosleep(0, TARGETING::is_vpo()
+                    ? TEN_CTX_SWITCHES_NS
+                    : wakeupIntervalNanoSecs);
+
+            // All targets' maint cmd should time out
+            while(++indexWkPr < stateMac.iv_workFlowProperties.size())
+            {
+                if((*stateMac.iv_workFlowProperties[indexWkPr]).status
+                    != COMMAND_TIMED_OUT)
+                {
+                    TS_FAIL("StateMachine:processCommandTimeout(1)."
+                            "Unexpected status");
+                }
+            }
+
+            stateMac.shutdown();
+            TS_TRACE(EXIT_MRK "testSmProcessCommandTimeout");
+
+        }
+
+        void testSmProcessCommandTimeoutSubset(void)
+        {
+            using namespace MDIA;
+            using namespace TARGETING;
+
+            TS_TRACE(ENTER_MRK "testSmProcessCommandTimeoutSubset");
+
+            FakeStateMachine2 stateMac;
+            WorkFlowAssocMap wkFlEntries;
+
+            TargetHandleList targets;
+            getTargets(3, targets);
+
+            wkFlEntries[targets[0]].push_back(START_SCRUB);
+            wkFlEntries[targets[1]].push_back(START_SCRUB);
+            wkFlEntries[targets[2]].push_back(START_SCRUB);
+            stateMac.setup(wkFlEntries);
+
+            uint64_t indexWkPr = stateMac.iv_workFlowProperties.size();
+            while(indexWkPr-- != 0)
+            {
+                stateMac.doMaintCommand(
+                *stateMac.iv_workFlowProperties[indexWkPr]);
+            }
+
+            // Send a 'COMMAND_COMPLETE' event for one of the targets
+            MaintCommandEvent event;
+            event.type = COMMAND_COMPLETE;
+            event.target = targets[0];
+            stateMac.processMaintCommandEvent(event);
+
+            static const uint64_t wakeupIntervalNanoSecs = 100000000;
+            nanosleep(0, TARGETING::is_vpo()
+                    ? TEN_CTX_SWITCHES_NS
+                    : wakeupIntervalNanoSecs);
+
+            indexWkPr = stateMac.iv_workFlowProperties.size();
+            indexWkPr--;
+
+            // One of the targets should not have timedout.
+            if((*stateMac.iv_workFlowProperties[indexWkPr]).status
+                == COMMAND_TIMED_OUT)
+            {
+                TS_FAIL("StateMachine:processCommandTimeout(2)."
+                        "Unexpected status");
+            }
+            else
+            {
+                //Rest of the targets should have timedout
+                while(indexWkPr-- != 0)
+                {
+                    if((*stateMac.iv_workFlowProperties[indexWkPr]).status
+                        != COMMAND_TIMED_OUT)
+                    {
+                        TS_FAIL("StateMachine:processCommandTimeout(3)."
+                                "Unexpected status");
+                    }
+                }
+            }
+
+            stateMac.shutdown();
+            TS_TRACE(EXIT_MRK "testSmProcessCommandTimeoutSubset");
+
+        }
 };
 #endif