MDIA: add MBA timeout count and check

Change-Id: I2116a10a577bd4ee66797f7f70947b2fa51babfb
CQ: SW244391
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/9237
Tested-by: Jenkins Server
Reviewed-by: Zane Shelley <zshelle@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

4 files changed, 40 insertions, 10 deletions

diff --git a/src/usr/diag/mdia/mdiafwd.H b/src/usr/diag/mdia/mdiafwd.H
index 164daafe7..c0b308dc6 100644
--- a/src/usr/diag/mdia/mdiafwd.H
+++ b/src/usr/diag/mdia/mdiafwd.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -102,6 +102,9 @@ enum
     IPOLL_STATUS = 0x01020014,
     PBUS_GP1 = 0x02000001,
     PBUS_GP2 = 0x02000002,
+    GLOBAL_CS_FIR = 0x570F001C,
+    GLOBAL_RE_FIR = 0x570F001B,
+    GLOBAL_SPA = 0x570F001A,
 };
 
 /**
diff --git a/src/usr/diag/mdia/mdiasm.C b/src/usr/diag/mdia/mdiasm.C
index 658390eb9..a2a5053fc 100644
--- a/src/usr/diag/mdia/mdiasm.C
+++ b/src/usr/diag/mdia/mdiasm.C
@@ -93,6 +93,9 @@ void addTimeoutFFDC(TargetHandle_t i_mba, errlHndl_t & io_log)
         IPOLL_STATUS,
         PBUS_GP1,
         PBUS_GP2,
+        GLOBAL_CS_FIR,
+        GLOBAL_RE_FIR,
+        GLOBAL_SPA,
     };
 
     // get the parent membuf
@@ -142,6 +145,7 @@ void addTimeoutFFDC(TargetHandle_t i_mba, errlHndl_t & io_log)
         {
             continue;
         }
+
         for(const uint64_t * regIt = tableIt->begin;
                 regIt != tableIt->end;
                 ++regIt)
@@ -275,14 +279,27 @@ void StateMachine::processCommandTimeout(const MonitorIDs & i_monitorIDs)
                 // Pending maint cmd complete, reset timer
                 if(mbaspa & ~mbaspamask)
                 {
-                    MDIA_FAST("sm: work item %d reset timed out on: %x",
-                        *((*wit)->workItem),
-                        get_huid(target));
-                    // register a new timeout monitor
-                    uint64_t monitorId = getMonitor().addMonitor(MBA_TIMEOUT);
-                    (*wit)->timer = monitorId;
-
-                    break;
+                    if((*wit)->timeoutCnt >= MBA_TIMEOUTCNT_MAX)
+                    {
+                        MDIA_FAST("sm: work item %d timed out on: %x, "
+                                  "timeoutCnt: %d", *((*wit)->workItem),
+                                  get_huid(target), (*wit)->timeoutCnt);
+                    }
+                    else
+                    {
+                        MDIA_FAST("sm: work item %d reset timed out on: %x, "
+                                  "timeoutCnt: %d", *((*wit)->workItem),
+                                  get_huid(target), (*wit)->timeoutCnt);
+                        // register a new timeout monitor
+                        uint64_t monitorId =
+                                 getMonitor().addMonitor(MBA_TIMEOUT);
+                        (*wit)->timer = monitorId;
+
+                        // advance timeout counter
+                        (*wit)->timeoutCnt++;
+
+                        break;
+                    }
                 }
 
                 // If maint cmd complete bit is not on, time out
@@ -435,6 +452,7 @@ void StateMachine::setup(const WorkFlowAssocMap & i_list)
         p->timer = 0;
         p->restartCommand = false;
         p->memSize = 0;
+        p->timeoutCnt = 0;
 
         // get the memsize attached to this mba
 
diff --git a/src/usr/diag/mdia/mdiasm.H b/src/usr/diag/mdia/mdiasm.H
index 37aa9f9a7..d5fc9950a 100644
--- a/src/usr/diag/mdia/mdiasm.H
+++ b/src/usr/diag/mdia/mdiasm.H
@@ -41,6 +41,9 @@ namespace MDIA
 //MBA timeout value - 30 secs
 static const uint64_t MBA_TIMEOUT = 30000000000;
 
+//MBA timeout count max before timing out (~1 min)
+static const uint64_t MBA_TIMEOUTCNT_MAX = 2;
+
 /**
  * @brief work flow phases
  */
diff --git a/src/usr/diag/mdia/mdiasmimpl.H b/src/usr/diag/mdia/mdiasmimpl.H
index d91c14d58..5a1bbe7bc 100644
--- a/src/usr/diag/mdia/mdiasmimpl.H
+++ b/src/usr/diag/mdia/mdiasmimpl.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -86,6 +86,12 @@ struct WorkFlowProperties
      * @brief mba position on membuf (0/1)
      */
     uint8_t chipUnit;
+
+    /**
+     * @brief timeout count associated with the workFlow
+     */
+    uint8_t timeoutCnt;
+
 };
 
 /**