Shutdown mailbox resources

Change-Id: Ie746c43c18e1e3606fd85cbf2a936b48026d62df
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1106
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

8 files changed, 286 insertions, 53 deletions

diff --git a/src/include/usr/initservice/initserviceif.H b/src/include/usr/initservice/initserviceif.H
index ba29d93a8..44a9c8961 100644
--- a/src/include/usr/initservice/initserviceif.H
+++ b/src/include/usr/initservice/initserviceif.H
@@ -35,6 +35,8 @@ namespace INITSERVICE
     enum EventPriority_t
     {
         LOWEST_PRIORITY = 0,    //!<< Notifiy last, LIFO order
+        INTR_PRIORITY   = LOWEST_PRIORITY, //!<< Shutdown INTR
+        MBOX_PRIORITY   = 1,    //!<< Shutdown MBOX
         NO_PRIORITY = 16,       //!<< No dependency / dont' care, LIFO order
         HIGHEST_PRIORITY = 127, //!<< Nofity first, LIFO order
     };
diff --git a/src/include/usr/mbox/mbox_reasoncodes.H b/src/include/usr/mbox/mbox_reasoncodes.H
index 3229ccc6f..46d0b4738 100644
--- a/src/include/usr/mbox/mbox_reasoncodes.H
+++ b/src/include/usr/mbox/mbox_reasoncodes.H
@@ -38,6 +38,7 @@ namespace MBOX
         MOD_MBOXREGISTER        = 0x06, // MailboxSp::msgq_register
         MOD_MBOXSRV_FSP_MSG     = 0x07, // MailboxSp::handle_hbmbox_msg
         MOD_MBOXSRV_SENDMSG     = 0x08, // MailboxSp::send_msg
+        MOD_MBOXSRV_INIT        = 0x09, // MailboxSp::_init
     };
 
     enum MBOXReasonCode
@@ -56,6 +57,8 @@ namespace MBOX
         RC_INVALID_MBOX_MSG_TYPE     = HBMBOX_COMP_ID | 0x0C,
         RC_MBOX_SERVICE_NOT_READY    = HBMBOX_COMP_ID | 0x0D,
         RC_INVALID_DMA_LENGTH        = HBMBOX_COMP_ID | 0x0E,
+        RC_KERNEL_REG_FAILED         = HBMBOX_COMP_ID | 0x0F,
+        RC_MAILBOX_DISABLED          = HBMBOX_COMP_ID | 0x10,
     };
 };
 
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 2683ca43b..562dcf99f 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -154,7 +154,7 @@ errlHndl_t IntrRp::_init()
         // Register event to be called on shutdown
         INITSERVICE::registerShutdownEvent(iv_msgQ,
                                            MSG_INTR_SHUTDOWN,
-                                           INITSERVICE::LOWEST_PRIORITY);
+                                           INITSERVICE::INTR_PRIORITY);
     }
 
     return err;
@@ -288,8 +288,8 @@ void IntrRp::msgHandler()
 
             case MSG_INTR_UNREGISTER_MSGQ:
                 {
-                    TRACDCOMP(g_trac_intr,
-                              "UNREG: msg type = 0x%lx",
+                    TRACFCOMP(g_trac_intr,
+                              "INTR remove registration of interrupt type = 0x%lx",
                               msg->data[0]);
 
                     msg_q_t msgQ = NULL;
@@ -412,7 +412,7 @@ errlHndl_t IntrRp::registerInterrupt(msg_q_t i_msgQ,
     Registry_t::iterator r = iv_registry.find(i_intr_type);
     if(r == iv_registry.end())
     {
-        TRACDCOMP(g_trac_intr,"INTR::register %x", i_intr_type);
+        TRACFCOMP(g_trac_intr,"INTR::register intr type 0x%x", i_intr_type);
         iv_registry[i_intr_type] = intr_response_t(i_msgQ,i_msg_type);
     }
     else
@@ -598,6 +598,7 @@ void IntrRp::shutDown()
             deconfigureInterruptPresenter(pir);
         }
     }
+    TRACFCOMP(g_trac_intr,INFO_MRK,"INTR is shutdown");
 }
 
 //----------------------------------------------------------------------------
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index e687a0a0e..ac7a50c3f 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -29,6 +29,7 @@
 #include "mboxdd.H"
 #include <sys/task.h>
 #include <initservice/taskargs.H>
+#include <initservice/initserviceif.H>
 #include <sys/vfs.h>
 #include <devicefw/userif.H>
 #include <mbox/mbox_reasoncodes.H>
@@ -60,8 +61,10 @@ MailboxSp::MailboxSp()
         iv_respondq(),
         iv_dmaBuffer(),
         iv_trgt(NULL),
+        iv_shutdown_msg(NULL),
         iv_rts(true),
-        iv_dma_pend(false)
+        iv_dma_pend(false),
+        iv_disabled(true)
 {
     // mailbox target
     TARGETING::targetService().masterProcChipTargetHandle(iv_trgt);
@@ -88,19 +91,50 @@ errlHndl_t MailboxSp::_init()
 {
     errlHndl_t err = NULL;
     size_t rc = 0;
+
     iv_msgQ = msg_q_create();
     rc = msg_q_register(iv_msgQ, VFS_ROOT_MSG_MBOX);
 
-    // Register to get interrupts for mailbox
-    err = INTR::registerMsgQ(iv_msgQ, MSG_INTR, INTR::FSP_MAILBOX);
+    if(rc)   // could not register msgQ with kernel
+    {
+        TRACFCOMP(g_trac_mbox,ERR_MRK "MailboxSP::_init() Could not register"
+                  "message qeueue with kernel");
 
-    if(!err)
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+         * @moduleid        MBOX::MOD_MBOXSRV_INIT
+         * @reasoncode      MBOX::RC_KERNEL_REG_FAILED
+         * @userdata1       rc from msq_q_register
+         * @defdesc         Could not register mailbox message queue
+         */
+        err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
+             MBOX::MOD_MBOXSRV_INIT,
+             MBOX::RC_KERNEL_REG_FAILED,    //  reason Code
+             rc,                        // rc from msg_send
+             0
+            );
+
+        return err;
+    }
+
+    // Initialize the mailbox hardware
+    err = mboxInit(iv_trgt);
+    if (err)
     {
-        err = mboxInit(iv_trgt);
+        return err;
+    }
 
-        task_create(MailboxSp::msg_handler, NULL);
+    // Register to get interrupts for mailbox
+    err = INTR::registerMsgQ(iv_msgQ, MSG_INTR, INTR::FSP_MAILBOX);
+    if(err)
+    {
+        return err;
     }
 
+    task_create(MailboxSp::msg_handler, NULL);
+
     // Send message to FSP on base DMA buffer zone
     msg_t * msg = msg_allocate();
     msg->type = MSG_INITIAL_DMA;
@@ -109,6 +143,14 @@ errlHndl_t MailboxSp::_init()
     msg->extra_data = NULL;
     MBOX::send(FSP_MAILBOX_MSGQ,msg);
 
+    iv_disabled = false;
+
+    // Register for shutdown
+    INITSERVICE::registerShutdownEvent(iv_msgQ,
+                                       MSG_MBOX_SHUTDOWN,
+                                       INITSERVICE::MBOX_PRIORITY);
+    
+
     return err;
 }
 
@@ -147,16 +189,7 @@ void MailboxSp::msgHandler()
             // Interrupt from the mailbox hardware
             case MSG_INTR:
                 {
-                    if(msg->data[0] != INTR::SHUT_DOWN)
-                    {
-                        err = handleInterrupt();
-                    }
-                    else
-                    {
-                        // Shutdown the mailbox
-                        TRACFCOMP(g_trac_mbox,"MBOXSP Shutdown event received");
-                        // TODO in story 41136
-                    }
+                    err = handleInterrupt();
 
                     // Respond to the interrupt handler regardless of err
                     msg->data[0] = 0;
@@ -172,42 +205,18 @@ void MailboxSp::msgHandler()
 
                         assert(0);
                     }
-                }
-
-                break;
-
-
-            case MSG_SEND:
-                {
-                    // Build mailbox message
-                    mbox_msg_t mbox_msg;
-                    mbox_msg.msg_queue_id = static_cast<uint32_t>(msg->data[0]);
-                    msg_t * payload = reinterpret_cast<msg_t*>(msg->extra_data);
-                    mbox_msg.msg_payload = *payload;  //copy in payload
 
-                    if(msg_is_async(msg))
+                    if(iv_shutdown_msg && quiesced())
                     {
-                        msg_free(payload);
-                        msg_free(msg);
+                        handleShutdown();
                     }
-                    else  //synchronous
-                    {
-                        msg->data[1] = 0;  // used later for return value
-
-                        // need to watch for a response
-                        response = new msg_respond_t(msg);
-
-                        // Convert a 64 bit pointer to a 32 bit unsigned int.
-                        mbox_msg.msg_id =
-                            static_cast<uint32_t>
-                            (reinterpret_cast<uint64_t>(response->key));
+                }
 
-                        iv_respondq.insert(response);
-                    }
+                break;
 
-                    send_msg(&mbox_msg);
 
-                }
+            case MSG_SEND:
+                handleNewMessage(msg);
                 break;
 
             case MSG_REGISTER_MSGQ:
@@ -230,6 +239,21 @@ void MailboxSp::msgHandler()
                 }
                 break;
 
+            case MSG_MBOX_SHUTDOWN:
+                {
+                    TRACFCOMP(g_trac_mbox,"MBOXSP Shutdown event received");
+
+                    iv_shutdown_msg = msg;      // Respond to this when done
+                    iv_disabled = true;         // stop incomming new messages
+
+                    if(quiesced())
+                    {
+                        handleShutdown();       // done - shutdown now.
+                    }
+                    // else wait for things to quiesce
+                }
+                break;
+
             default: //Huh? Ignore it.
 
                 TRACFCOMP(g_trac_mbox, ERR_MRK "MailboxSp::msgHandler() "
@@ -263,6 +287,73 @@ void MailboxSp::msgHandler()
 }
 
 
+void MailboxSp::handleNewMessage(msg_t * i_msg)
+{
+    // Build mailbox message
+    mbox_msg_t mbox_msg;
+    mbox_msg.msg_queue_id = static_cast<uint32_t>(i_msg->data[0]);
+    msg_t * payload = reinterpret_cast<msg_t*>(i_msg->extra_data);
+    mbox_msg.msg_payload = *payload;  //copy in payload
+
+    if(msg_is_async(i_msg))
+    {
+        msg_free(payload);
+        msg_free(i_msg);
+    }
+
+    if(iv_disabled)
+    {
+        TRACFCOMP(g_trac_mbox,WARN_MRK
+                  "MSGSEND - mailboxsp is disabled. Message dropped!"
+                  " msgQ=0x%x type=0x%x",
+                  mbox_msg.msg_queue_id,
+                  mbox_msg.msg_payload.type);
+
+        if(!msg_is_async(i_msg)) // synchronous
+        {
+            /*@ errorlog tag
+             * @errortype       ERRL_SEV_INFORMATIONAL
+             * @moduleid        MOD_MBOXSRV_SENDMSG
+             * @reasoncode      RC_MAILBOX_DISABLED
+             * @userdata1       queue_id
+             * @userdata2       message type
+             * @defdesc         Mailbox is disabled, message dropped.
+             */
+            errlHndl_t err = new ERRORLOG::ErrlEntry
+                (
+                 ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                 MBOX::MOD_MBOXSRV_SENDMSG,
+                 MBOX::RC_MAILBOX_DISABLED,        //  reason Code
+                 i_msg->data[0],                   // queue id 
+                 payload->type                     // message type
+                );
+
+            i_msg->data[1] = reinterpret_cast<uint64_t>(err);
+
+            msg_respond(iv_msgQ,i_msg);
+        }
+    }
+    else
+    {
+
+        if(!msg_is_async(i_msg))  //synchronous
+        {
+            i_msg->data[1] = 0;  // used later for return value
+
+            // need to watch for a response
+            msg_respond_t * response = new msg_respond_t(i_msg);
+
+            // Convert a 64 bit pointer to a 32 bit unsigned int.
+            mbox_msg.msg_id =
+                static_cast<uint32_t>
+                (reinterpret_cast<uint64_t>(response->key));
+
+            iv_respondq.insert(response);
+        }
+
+        send_msg(&mbox_msg);
+    }
+}
 
 // Note: When called due to an ACK or retry, iv_rts should be true.
 void MailboxSp::send_msg(mbox_msg_t * i_msg)
@@ -1050,6 +1141,25 @@ errlHndl_t MailboxSp::handleInterrupt()
 }
 
 
+void MailboxSp::handleShutdown()
+{
+    // Shutdown the hardware
+    errlHndl_t err = mboxddShutDown(iv_trgt);
+
+    if(err)  // SCOM failed.
+    {
+        // If this failed, the whole system is probably buggered up.
+        errlCommit(err,HBMBOX_COMP_ID);
+        assert(0);
+    }
+
+    INTR::unRegisterMsgQ(INTR::FSP_MAILBOX);
+
+    msg_respond(iv_msgQ,iv_shutdown_msg);
+    TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is shutdown");
+}
+
+
 // ----------------------------------------------------------------------------
 // External Interfaces @see mboxif.H
 // ----------------------------------------------------------------------------
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 0980940c2..cdcbac331 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -56,6 +56,7 @@ namespace MBOX
         MSG_REGISTER_MSGQ,
         MSG_UNREGISTER_MSGQ,
         MSG_INTR,
+        MSG_MBOX_SHUTDOWN,
     };
 
 
@@ -193,6 +194,31 @@ namespace MBOX
              */
             void handle_hbmbox_resp(mbox_msg_t & i_mbox_msg);
 
+            /**
+             * Handle a new message from HB to FSP
+             * @param[in] i_msg, The carrier message
+             */
+            void handleNewMessage(msg_t* i_msg);
+
+            /**
+             * Handle shutdown
+             */
+            void handleShutdown();
+
+            /**
+             * Query Quiesced
+             * @returns [true|false]
+             */
+            ALWAYS_INLINE
+            bool quiesced()
+            {
+                return (iv_rts &&
+                        !iv_dma_pend &&
+                        iv_sendq.empty() &&
+                        iv_respondq.empty());
+            }
+
+
             enum
             {
                 MAX_RETRY_COUNT = 3,
@@ -232,11 +258,12 @@ namespace MBOX
             mbox_msg_t  iv_msg_to_send; //!< message being sent
             respond_q_t iv_respondq;    //!< msg respond pending list
             registry_t  iv_registry;    //!< Registered queue
-
             DmaBuffer   iv_dmaBuffer;   //!< DMA buffer manager
             TARGETING::Target * iv_trgt;//!< mailbox device driver target
+            msg_t *     iv_shutdown_msg;//!< Message to shutdown mbox
             bool        iv_rts;         //!< ready to send flag
             bool        iv_dma_pend;    //!< Request pending for more DMA buffers
+            bool        iv_disabled;    //!< Mailboxsp is shut off
     };
 };
 
diff --git a/src/usr/mbox/mboxdd.C b/src/usr/mbox/mboxdd.C
index 7391564e2..61740041c 100644
--- a/src/usr/mbox/mboxdd.C
+++ b/src/usr/mbox/mboxdd.C
@@ -513,6 +513,7 @@ errlHndl_t mboxWrite(TARGETING::Target* i_target,void* i_buffer,
     return l_err;
 }
 
+
 /**
  * @brief Reads the mailbox PIB error status register
  */
@@ -602,4 +603,69 @@ errlHndl_t mboxInit(TARGETING::Target* i_target)
 }
 
 
+errlHndl_t mboxddMaskInterrupts(TARGETING::Target * i_target)
+{
+    errlHndl_t err = NULL;
+
+    // Mask off all interrupts
+    // Reset intr enable bits by setting the bits in MBOX_DB_INT_MASK_PIB_RC
+    uint64_t scom_data = (static_cast<uint64_t>(MBOX_DOORBELL_ERROR) |
+                          static_cast<uint64_t>(MBOX_HW_ACK) |
+                          static_cast<uint64_t>(MBOX_DATA_PENDING)) << 32;
+
+    size_t scom_len = sizeof(uint64_t);
+
+    err = deviceOp(DeviceFW::WRITE,
+                   i_target,
+                   reinterpret_cast<void*>(&scom_data),
+                   scom_len,
+                   DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_MASK_PIB_RC));
+
+    return err;
+}
+
+
+errlHndl_t mboxddShutDown(TARGETING::Target* i_target)
+{
+    size_t scom_len = sizeof(uint64_t);
+    uint64_t scom_data = 0;
+
+    errlHndl_t err = mboxddMaskInterrupts(i_target);
+
+    if(!err)
+    {
+        // Clear the status reg
+        //Turn off Permission to Send
+        //Turn off everything possible
+        err = deviceOp(DeviceFW::WRITE,
+                       i_target,
+                       reinterpret_cast<void*>(&scom_data),
+                       scom_len,
+                       DEVICE_XSCOM_ADDRESS(MBOX_DB_STAT_CNTRL_1));
+    }
+
+    // Clear any pending stuff
+    if(!err)
+    {
+        err = deviceOp(DeviceFW::READ,
+                       i_target,
+                       reinterpret_cast<void*>(&scom_data),
+                       scom_len,
+                       DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_REG_PIB));
+    }
+
+    if(!err)
+    {
+        err = deviceOp(DeviceFW::WRITE,
+                       i_target,
+                       reinterpret_cast<void*>(&scom_data),
+                       scom_len,
+                       DEVICE_XSCOM_ADDRESS(MBOX_DB_INT_REG_PIB));
+    }
+
+    // TODO Others?
+
+    return err;
+}
+
 }; //end MBOX namespace
diff --git a/src/usr/mbox/mboxdd.H b/src/usr/mbox/mboxdd.H
index ac5063d14..29851bcc6 100644
--- a/src/usr/mbox/mboxdd.H
+++ b/src/usr/mbox/mboxdd.H
@@ -54,8 +54,28 @@ namespace MBOX
         MBOX_PARITY_ERR      = 0x01000000, /* LBUS RAM Parity Error Detected */
     };
 
+    /**
+     * @brief Initialize device driver hardware
+     * 
+     * @param[in] i_target, Chip target of the MBOX operation
+     * @return errlHndl_t If scom error | NULL (success)
+     */
      errlHndl_t mboxInit(TARGETING::Target* i_target);
 
+     /**
+      * @brief Disable the reporting of interrupts
+      * @param[in] i_target, Chip target of the MBOX operations
+      */
+     errlHndl_t mboxddMaskInterrupts(TARGETING::Target * i_target);
+
+     /**
+      * @brief Shutdown device driver
+      *
+      * @param[in] i_target, Chip target of the MBOX operation
+      * @return errlHndl_t if scom error | NULL (success)
+      */
+     errlHndl_t mboxddShutDown(TARGETING::Target* i_target);
+
     /**
      * @brief Performs a mailbox read operation
      *
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index 080f21ccf..ad6865d2e 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -50,7 +50,9 @@ class MboxSPTest : public CxxTest::TestSuite
        */
       void testSendAsync(void)
       {
-          // TODO Temporaritly DISABLE in VBU until INTR BAR is set
+          //  This testcase only works if there is no FSP.
+          //  Must be disabled for vpo and
+          //  TODO disble if there is an FSP.
           if( TARGETING::is_vpo() )
           {
               return;
@@ -145,7 +147,9 @@ class MboxSPTest : public CxxTest::TestSuite
        */
       void testSendSync(void)
       {
-          // TODO Temporaritly DISABLE in VBU until INTR BAR is set
+          //  This testcase only works if there is no FSP.
+          //  Must be disabled for vpo and
+          //  TODO disble if there is an FSP.
           if( TARGETING::is_vpo() )
           {
               return;