Mailbox suspend/resume for multinode IPL

RTC: 63126
Change-Id: I590fe2d38eaef58ffa8e664ca202bd7fc9877b16
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/4551
Tested-by: Jenkins Server
Reviewed-by: ADAM R. MUHLE <armuhle@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

5 files changed, 239 insertions, 8 deletions

diff --git a/src/include/usr/mbox/mbox_reasoncodes.H b/src/include/usr/mbox/mbox_reasoncodes.H
index 4760d1de1..2bce46660 100644
--- a/src/include/usr/mbox/mbox_reasoncodes.H
+++ b/src/include/usr/mbox/mbox_reasoncodes.H
@@ -42,6 +42,7 @@ namespace MBOX
         MOD_MBOXSRC_UNCLAIMED   = 0x0A, // MailboxSp::handleUnclaimed
         MOD_MBOX_SEND           = 0x0B, // MBOX::send
         MOD_MBOXSRV_IPC_MSG     = 0x0C, // MailboxSp::handleIPC
+        MOD_MBOX_MSGQ_FAIL      = 0x0D, // MBOX::makeErrlMsgQSendFail
     };
 
     enum MBOXReasonCode
diff --git a/src/include/usr/mbox/mboxif.H b/src/include/usr/mbox/mboxif.H
index a8609a31f..8bce7b495 100644
--- a/src/include/usr/mbox/mboxif.H
+++ b/src/include/usr/mbox/mboxif.H
@@ -108,6 +108,26 @@ namespace MBOX
      */
     bool mailbox_enabled();
 
+    /**
+     * Suspend the mailbox. 
+     *
+     * @return error handle on error
+     *
+     * @note:
+     *     Any message sent to the FSP will be queued and send after the mailbox
+     *     is resumed. Interrupts from the FSP MBox will be masked.
+     */
+    errlHndl_t suspend();
+
+    /**
+     * Resume the mailbox
+     *
+     * @return error handle on error
+     *
+     * @note  FSP Mbox re-enabled, queued messages sent.
+     */
+    errlHndl_t resume();
+
 }; // end namespace MBOX
 
 #endif
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 3add7caf9..0af678a9f 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -40,6 +40,12 @@
 
 #define MBOX_TRACE_NAME MBOX_COMP_NAME
 
+// Local functions
+namespace MBOX
+{
+    errlHndl_t makeErrlMsgQSendFail(uint64_t i_errno);
+};
+
 using namespace MBOX;
 
 // Defined in mboxdd.C
@@ -69,7 +75,8 @@ MailboxSp::MailboxSp()
         iv_shutdown_msg(NULL),
         iv_rts(true),
         iv_dma_pend(false),
-        iv_disabled(true)
+        iv_disabled(true),
+        iv_suspended(false)
 {
     // mailbox target
     TARGETING::targetService().masterProcChipTargetHandle(iv_trgt);
@@ -246,6 +253,11 @@ void MailboxSp::msgHandler()
                     {
                         handleShutdown();
                     }
+
+                    if(iv_suspended && quiesced())
+                    {
+                        suspend();
+                    }
                 }
 
                 break;
@@ -282,6 +294,11 @@ void MailboxSp::msgHandler()
                     iv_shutdown_msg = msg;      // Respond to this when done
                     iv_disabled = true;         // stop incomming new messages
 
+                    if(iv_suspended == true)
+                    {
+                        resume();
+                    }
+
                     handleUnclaimed();
                     if(quiesced())
                     {
@@ -291,6 +308,38 @@ void MailboxSp::msgHandler()
                 }
                 break;
 
+            case MSG_MBOX_SUSPEND:
+
+                if(!iv_disabled)
+                {
+                    TRACFCOMP(g_trac_mbox,
+                              INFO_MRK"MBOXSP Suspend event received");
+
+                    iv_suspend_msg = msg;   // Respond to this when done
+                    iv_suspended = true;    // Queue any new messages
+
+                    if(quiesced())
+                    {
+                        suspend();    // suspended
+                    }
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_mbox,
+                              INFO_MRK"MBOXSP Ignored request to suspend a"
+                              " disabled mailbox");
+                    msg_respond(iv_msgQ,msg);
+                }
+                break;
+
+            case MSG_MBOX_RESUME:
+
+                resume();
+                msg_respond(iv_msgQ,msg);
+                TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox function resumed");
+
+                break;
+
             case MSG_IPC: // Interprocessor Messages
                 // Look for IPC message
                 // If not, just ignore
@@ -430,12 +479,13 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
     // Can't send now if:
     //  - busy (waiting for ACK)
     //  - a DMA buffer request is pending
+    //  - The mailbox is suspended
     //  - there is nothing to send
     //
-    //  TODO future optimization: If both iv_rts and iv_dma_pend are true then
-    //  could look for a mesaage in the sendq that does not require a DMA
-    //  buffer and send it.
-    if(!iv_rts || iv_dma_pend || iv_sendq.size() == 0)
+    //  Future enhancement: If both iv_rts and iv_dma_pend are true then
+    //  could look for a mesaage from a different msgq in the sendq that does
+    //  not require a DMA buffer and send it.
+    if(!iv_rts || iv_dma_pend || iv_suspended || iv_sendq.size() == 0)
     {
         return;
     }
@@ -1523,7 +1573,8 @@ void MailboxSp::handleShutdown()
 
         errlCommit(err,MBOX_COMP_ID);
 
-        TRACFCOMP(g_trac_mbox, ERR_MRK"MBOXSP HALTED on critical error!");
+        TRACFCOMP(g_trac_mbox,
+                  ERR_MRK"MBOXSP Shutdown. HALTED on critical error!");
         crit_assert(0);
     }
 
@@ -1531,6 +1582,46 @@ void MailboxSp::handleShutdown()
     TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is shutdown");
 }
 
+void MailboxSp::suspend()
+{
+    // Mask interrupts in the mbox hardware
+    errlHndl_t err = mboxddMaskInterrupts(iv_trgt);
+    if(err)  // SCOM failed.
+    {
+        // If this failed, the whole system is probably buggered up.
+
+        errlCommit(err,MBOX_COMP_ID);
+
+        TRACFCOMP(g_trac_mbox,
+                  ERR_MRK"MBOXSP Suspend. HALTED on critical error!");
+        crit_assert(0);
+    }
+
+    msg_respond(iv_msgQ,iv_suspend_msg);
+    TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is suspended");
+}
+
+void MailboxSp::resume()
+{
+    iv_suspended = false;
+
+    if(!iv_disabled)
+    {
+        // Enable the mbox hardware
+        errlHndl_t err = mboxInit(iv_trgt);
+        if(err)  // SCOM failed.
+        {
+            // If this failed, the whole system is probably buggered up.
+
+            errlCommit(err,MBOX_COMP_ID);
+
+            TRACFCOMP(g_trac_mbox,
+                      ERR_MRK"MBOXSP Resume. HALTED on critical error!");
+            crit_assert(0);
+        }
+        send_msg();   // send next message on queue
+    }
+}
 
 // ----------------------------------------------------------------------------
 // External Interfaces @see mboxif.H
@@ -1726,4 +1817,89 @@ bool MBOX::mailbox_enabled()
     return enabled;
 }
 
+errlHndl_t MBOX::suspend()
+{
+    errlHndl_t err = NULL;
+    msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
+    if(mboxQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->type = MSG_MBOX_SUSPEND;
 
+        int rc = msg_sendrecv(mboxQ, msg);
+
+        if (rc)
+        {
+            TRACFCOMP(g_trac_mbox, ERR_MRK
+                      "MBOX::suspend msg_sendrecv() failed. errno = %d",
+                      rc);
+
+            err = makeErrlMsgQSendFail(rc);
+        }
+
+        msg_free(msg);
+    }
+    else
+    {
+        TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available");
+    }
+
+    return err;
+}
+
+errlHndl_t MBOX::resume()
+{
+    errlHndl_t err = NULL;
+    msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
+    if(mboxQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->type = MSG_MBOX_RESUME;
+
+        int rc = msg_sendrecv(mboxQ, msg);
+
+        if (rc)
+        {
+            TRACFCOMP(g_trac_mbox, ERR_MRK
+                      "MBOX::resume msg_sendrecv failed. errno = %d",
+                      rc);
+            err = makeErrlMsgQSendFail(rc);
+        }
+
+        msg_free(msg);
+    }
+    else
+    {
+        TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available");
+    }
+
+    return err;
+}
+
+errlHndl_t MBOX::makeErrlMsgQSendFail(uint64_t i_errno)
+{
+        TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available");
+
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_INFORMATIONAL
+         * @moduleid        MBOX::MOD_MBOX_MSGQ_FAIL
+         * @reasoncode      MBOX::RC_MSG_SEND_ERROR
+         * @userdata1       kernel errno
+         * @userdata2       <unused>
+         *
+         * @devdesc         Message send to mailbox sp failed
+         *
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_INFORMATIONAL,    // severity
+             MBOX::MOD_MBOX_MSGQ_FAIL,            // moduleid
+             MBOX::RC_MSG_SEND_ERROR,             // reason code
+             i_errno,
+             0
+            );
+
+        err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,                                                                                               HWAS::SRCI_PRIORITY_HIGH);
+        
+        return err;
+}
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 7a87ed871..f7f205e96 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -57,6 +57,8 @@ namespace MBOX
         MSG_UNREGISTER_MSGQ,
         MSG_INTR,
         MSG_MBOX_SHUTDOWN,
+        MSG_MBOX_SUSPEND,
+        MSG_MBOX_RESUME,
         MSG_IPC,
     };
 
@@ -214,10 +216,22 @@ namespace MBOX
 
             /**
              * Handle shutdown
+             * @pre quiesced() == true
              */
             void handleShutdown();
 
             /**
+             * Perform the final actions needed to suspend the mailbox sp
+             * @pre quiesced() == true
+             */
+            void suspend();
+
+            /**
+             * Resume the mailbox from the suspend state
+             */
+            void resume();
+
+            /**
              * Send message to FSP indicating an invalid or undeliverable
              * message queue.
              * @param[in] i_mbox_msg.  The invalid message
@@ -282,9 +296,11 @@ namespace MBOX
 
             TARGETING::Target * iv_trgt;//!< mailbox device driver target
             msg_t *     iv_shutdown_msg;//!< Message to shutdown mbox
+            msg_t *     iv_suspend_msg; //!< Message to suspend mbox
             bool        iv_rts;         //!< ready to send flag
-            bool        iv_dma_pend;    //!< Request pending for more DMA buffers
-            bool        iv_disabled;    //!< Mailboxsp is shut off
+            bool        iv_dma_pend;    //!< Request pending for more DMA bufs
+            bool        iv_disabled;    //!< Mailboxsp shut off (rejects new)
+            bool        iv_suspended;   //!< Mailbox is suspended (queues new)
     };
 };
 
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index e3d546e5e..61a5c6527 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -242,6 +242,24 @@ class MboxSPTest : public CxxTest::TestSuite
           msg_free(msg);
       }
 
+      void testSuspendResume(void)
+      {
+
+          errlHndl_t err = MBOX::suspend();
+          if (err)
+          {
+              TS_FAIL("MBOX:suspend failed");
+              delete err;
+          }
+
+          err = MBOX::resume();
+          if (err)
+          {
+              TS_FAIL("MBOX::resume failed");
+              delete err;
+          }
+      }
+
       void testInvalidMsg(void)
       {
           msg_t * msg = msg_allocate();