Workaround SBE interupt injection bug on multi-node

Change-Id: Ib17ae25542100c88fd5e85add362876582ea4449
CQ: SW274682
Backport: release-fips820
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/13224
Tested-by: Jenkins Server
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

4 files changed, 79 insertions, 17 deletions

diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index abcc93b76..bb7b01513 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -81,6 +81,8 @@ MailboxSp::MailboxSp()
         iv_dma_pend(false),
         iv_disabled(true),
         iv_suspended(false),
+        iv_suspend_intr(false),
+        iv_allow_blk_resp(false),
         iv_sum_alloc(0),
         iv_pend_alloc(),
         iv_allocAddrs()
@@ -364,6 +366,8 @@ void MailboxSp::msgHandler()
 
                     iv_suspend_msg = msg;   // Respond to this when done
                     iv_suspended = true;    // Queue any new messages
+                    iv_suspend_intr = static_cast<bool>(msg->data[0]);
+                    iv_allow_blk_resp = static_cast<bool>(msg->data[1]);
 
                     if(quiesced())
                     {
@@ -1488,12 +1492,19 @@ bool MailboxSp::quiesced()
 {
     bool result = iv_rts && !iv_dma_pend && iv_sendq.empty();
 
+    TRACFCOMP(g_trac_mbox,INFO_MRK"Checking quiesced.. rts[%d] dma_pend[%d] sendq[%d]",
+              iv_rts, iv_dma_pend, iv_sendq.empty());
+
+
     if( result == true )
     {
+        TRACFCOMP(g_trac_mbox,INFO_MRK"quiesed == true, iv_shutdown_msg[%p]",
+                  iv_shutdown_msg);
         if(iv_shutdown_msg == NULL ||
            (iv_shutdown_msg->data[1] == SHUTDOWN_STATUS_GOOD))
         {
-            result = result && iv_respondq.empty();
+            //Check that respond q is empty OR don't care
+            result = result && (iv_respondq.empty() || iv_allow_blk_resp);
         }
         else // mbox is shutting down and system status is bad
         {
@@ -1524,6 +1535,8 @@ bool MailboxSp::quiesced()
         }
     }
 
+    TRACFCOMP(g_trac_mbox,INFO_MRK"Quiesced [%d]", result);
+
     return result;
 }
 
@@ -1736,6 +1749,19 @@ void MailboxSp::handleShutdown()
 
 void MailboxSp::suspend()
 {
+    TRACFCOMP(g_trac_mbox,INFO_MRK"Entering suspended");
+    if(iv_suspend_intr)
+    {
+        errlHndl_t err = mboxddMaskInterrupts(iv_trgt);
+        if(err)  // SCOM failed.
+        {
+            // If this failed, the whole system is probably buggered up.
+            errlCommit(err,MBOX_COMP_ID);
+            TRACFCOMP(g_trac_mbox,
+                      ERR_MRK"MBOXSP suspend HALTED on critical error!");
+            crit_assert(0);
+        }
+    }
     msg_respond(iv_msgQ,iv_suspend_msg);
     TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is suspended");
 }
@@ -1743,9 +1769,25 @@ void MailboxSp::suspend()
 void MailboxSp::resume()
 {
     iv_suspended = false;
+    iv_allow_blk_resp = false;
 
     if(!iv_disabled)
     {
+        //If interrupts were disabled, re-enable
+        if(iv_suspend_intr)
+        {
+            errlHndl_t err = mboxddEnableInterrupts(iv_trgt);
+            if(err)  // SCOM failed.
+            {
+                // If this failed, the whole system is probably buggered up.
+                errlCommit(err,MBOX_COMP_ID);
+                TRACFCOMP(g_trac_mbox,
+                          ERR_MRK"MBOXSP resume HALTED on critical error!");
+                crit_assert(0);
+            }
+            iv_suspend_intr = false;
+        }
+
         send_msg();   // send next message on queue
     }
 }
@@ -2042,7 +2084,7 @@ bool MBOX::mailbox_enabled()
     return enabled;
 }
 
-errlHndl_t MBOX::suspend()
+errlHndl_t MBOX::suspend(bool i_disable_hw_int, bool i_allow_resp)
 {
     errlHndl_t err = NULL;
     msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
@@ -2050,6 +2092,8 @@ errlHndl_t MBOX::suspend()
     {
         msg_t * msg = msg_allocate();
         msg->type = MSG_MBOX_SUSPEND;
+        msg->data[0] = static_cast<uint64_t>(i_disable_hw_int);
+        msg->data[1] = static_cast<uint64_t>(i_allow_resp);
 
         int rc = msg_sendrecv(mboxQ, msg);
 
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index c643fbe6b..caec9cc3f 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -328,6 +328,8 @@ namespace MBOX
             bool        iv_dma_pend;    //!< Request pending for more DMA bufs
             bool        iv_disabled;    //!< Mailboxsp shut off (rejects new)
             bool        iv_suspended;   //!< Mailbox is suspended (queues new)
+            bool        iv_suspend_intr;//!< Disable HW interrupts on suspend
+            bool        iv_allow_blk_resp;//!< Disable HW interrupts on suspend
             uint64_t    iv_sum_alloc;   //!< Total extra_data storage allocated
             msg_list_t  iv_pend_alloc;  //!< Pending memory allocations
             addr_list_t iv_allocAddrs;  //!< memory addresses allocated by mbox
diff --git a/src/usr/mbox/mboxdd.C b/src/usr/mbox/mboxdd.C
index 5ad5ad45c..63bca0ffd 100644
--- a/src/usr/mbox/mboxdd.C
+++ b/src/usr/mbox/mboxdd.C
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -605,43 +607,49 @@ errlHndl_t mboxGetErrStat(TARGETING::Target* i_target,uint64_t &o_status)
 
 errlHndl_t mboxInit(TARGETING::Target* i_target)
 {
+    //For now init only enables the interrupts
+    return mboxddEnableInterrupts(i_target);
+}
+
+
+errlHndl_t mboxddMaskInterrupts(TARGETING::Target * i_target)
+{
     errlHndl_t err = NULL;
-    size_t scom_len = sizeof(uint64_t);
 
-    // Setup mailbox intr mask reg
-    // Set bits 2,1,0
-    // assume we always use mailbox 1
+    // Mask off all interrupts
+    // Reset intr enable bits by setting the bits in MBOX_DB_INT_MASK_PIB_RC
     uint64_t scom_data = (static_cast<uint64_t>(MBOX_DOORBELL_ERROR) |
                           static_cast<uint64_t>(MBOX_HW_ACK) |
                           static_cast<uint64_t>(MBOX_DATA_PENDING)) << 32;
 
+    size_t scom_len = sizeof(uint64_t);
+
     err = deviceOp(DeviceFW::WRITE,
                    i_target,
                    reinterpret_cast<void*>(&scom_data),
                    scom_len,
-                   DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_MASK_PIB_RS));
+                   DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_MASK_PIB_RC));
+
     return err;
 }
 
-
-errlHndl_t mboxddMaskInterrupts(TARGETING::Target * i_target)
+errlHndl_t mboxddEnableInterrupts(TARGETING::Target * i_target)
 {
     errlHndl_t err = NULL;
+    size_t scom_len = sizeof(uint64_t);
 
-    // Mask off all interrupts
-    // Reset intr enable bits by setting the bits in MBOX_DB_INT_MASK_PIB_RC
+    // Setup mailbox intr mask reg
+    // Set bits 2,1,0
+    // assume we always use mailbox 1
     uint64_t scom_data = (static_cast<uint64_t>(MBOX_DOORBELL_ERROR) |
                           static_cast<uint64_t>(MBOX_HW_ACK) |
                           static_cast<uint64_t>(MBOX_DATA_PENDING)) << 32;
 
-    size_t scom_len = sizeof(uint64_t);
-
     err = deviceOp(DeviceFW::WRITE,
                    i_target,
                    reinterpret_cast<void*>(&scom_data),
                    scom_len,
-                   DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_MASK_PIB_RC));
-
+                   DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_MASK_PIB_RS));
     return err;
 }
 
diff --git a/src/usr/mbox/mboxdd.H b/src/usr/mbox/mboxdd.H
index e5c1cb9cb..86fb1ee80 100644
--- a/src/usr/mbox/mboxdd.H
+++ b/src/usr/mbox/mboxdd.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -69,6 +71,12 @@ namespace MBOX
      errlHndl_t mboxddMaskInterrupts(TARGETING::Target * i_target);
 
      /**
+      * @brief Enable the reporting of interrupts
+      * @param[in] i_target, Chip target of the MBOX operations
+      */
+     errlHndl_t mboxddEnableInterrupts(TARGETING::Target * i_target);
+
+     /**
       * @brief Shutdown device driver
       *
       * @param[in] i_target, Chip target of the MBOX operation