Clean handling of shutdown path in error cases

Change-Id: Id03c0f2fc91a31b05572222a2a2743db1518198c
RTC: 109993
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/11636
Tested-by: Jenkins Server
Reviewed-by: Michael Baiocchi <baiocchi@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: Corey V. Swenson <cswenson@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

4 files changed, 56 insertions, 13 deletions

diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 909774a84..af3c3fc9b 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -688,7 +688,7 @@ void IntrRp::msgHandler()
             case MSG_INTR_SHUTDOWN:
                 {
                     TRACFCOMP(g_trac_intr,"Shutdown event received");
-                    shutDown();
+                    shutDown(msg->data[0]);
 
                     msg_respond(iv_msgQ, msg);
 
@@ -1290,7 +1290,7 @@ errlHndl_t IntrRp::checkAddress(uint64_t i_addr)
     return err;
 }
 
-void IntrRp::shutDown()
+void IntrRp::shutDown(uint64_t i_status)
 {
     errlHndl_t err = NULL;
     msg_t * rmsg = msg_allocate();
@@ -1304,7 +1304,7 @@ void IntrRp::shutDown()
 
         rmsg->type = r->second.msgType;
         rmsg->data[0] = SHUT_DOWN;
-        rmsg->data[1] = 0;
+        rmsg->data[1] = i_status;
         rmsg->extra_data = NULL;
 
         int rc = msg_sendrecv(msgQ,rmsg);
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index d7ca8e10c..8a7360828 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -527,8 +527,12 @@ namespace INTR
 
             /**
              * Shutdown procedure
+             * @param[in] shutdown status.
+             *            Set to SHUTDOWN_STATUS_GOOD on a clean shutdown,
+             *            Otherwise is plid or other ShutdownStatus enum.
+             *            @see ShutdownStatus enums in sys/misc.h
              */
-            void shutDown();
+            void shutDown(uint64_t i_status);
 
             /**
              * Wait for all nodes to disable interrupts on MPIPL
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 90d5ab8c5..1c293c4ea 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -43,6 +43,7 @@
 #include <kernel/ipc.H>
 #include <arch/ppc.H>
 #include <errl/errlmanager.H>
+#include <sys/misc.h>
 
 // Local functions
 namespace MBOX
@@ -307,7 +308,9 @@ void MailboxSp::msgHandler()
 
             case MSG_MBOX_SHUTDOWN:
                 {
-                    TRACFCOMP(g_trac_mbox,"MBOXSP Shutdown event received");
+                    TRACFCOMP(g_trac_mbox,
+                              "MBOXSP Shutdown event received. Status 0x%x",
+                              msg->data[1]);
 
                     iv_shutdown_msg = msg;      // Respond to this when done
                     iv_disabled = true;         // stop incomming new messages
@@ -1471,6 +1474,49 @@ errlHndl_t MailboxSp::handleInterrupt()
     return err;
 }
 
+bool MailboxSp::quiesced()
+{
+    bool result = iv_rts && !iv_dma_pend && iv_sendq.empty();
+
+    if( result == true )
+    {
+        if(iv_shutdown_msg == NULL ||
+           (iv_shutdown_msg->data[1] == SHUTDOWN_STATUS_GOOD))
+        {
+            result = result && iv_respondq.empty();
+        }
+        else // mbox is shutting down and system status is bad
+        {
+            // Wait for HB to FSP sync message to complete.
+            // Don't wait for FSP to HB sync message to complete
+            msg_respond_t * resp = iv_respondq.begin();
+            while(resp)
+            {
+                if( resp->msg_queue_id >= MBOX::FSP_FIRST_MSGQ)
+                {
+                    // stil have pending HB->FSP sync messages
+                    result = false;
+                    break;
+                }
+                resp = resp->next;
+            }
+            // if result is still true then remaining responds, if any,
+            // should be ignored - mbox is quiesced.
+            if(result)
+            {
+                while(!iv_respondq.empty())
+                {
+                    resp = iv_respondq.begin();
+                    iv_respondq.erase(resp);
+                    delete resp;
+                }
+            }
+        }
+    }
+
+    return result;
+}
+
 void MailboxSp::handleIPC(queue_id_t i_msg_q_id, msg_t * i_msg)
 {
 
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 5998aa0a5..b9bcb1b2c 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -252,14 +252,7 @@ namespace MBOX
              * Query Quiesced
              * @returns [true|false]
              */
-            ALWAYS_INLINE
-            bool quiesced()
-            {
-                return (iv_rts &&
-                        !iv_dma_pend &&
-                        iv_sendq.empty() &&
-                        iv_respondq.empty());
-            }
+            bool quiesced();
 
 
             enum