Shutdown interrupt sources

RTC: 39878

Change-Id: I0c37a0dce8b818cf4a7b6fefe982b838dafcf917
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1074
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

9 files changed, 338 insertions, 257 deletions

diff --git a/src/include/sys/msg.h b/src/include/sys/msg.h
index 91dd51ce9..4bbf0b265 100644
--- a/src/include/sys/msg.h
+++ b/src/include/sys/msg.h
@@ -61,7 +61,6 @@ enum msg_sys_types_t
 
     MSG_INTR_EXTERN,    //!< Msg sent from kernel to user space on ext intr
     MSG_INTR_ADD_CPU,   //!< Add cpu core, data[0] = cpuid (PIR)
-    MSG_INTR_REMOVE_CPU,//!< un-configure intrp on cpu core, data[0] = cpuid
 };
 
 // System-defined root queue types
diff --git a/src/include/usr/intr/interrupt.H b/src/include/usr/intr/interrupt.H
index 5d0633051..f87a5cb5d 100644
--- a/src/include/usr/intr/interrupt.H
+++ b/src/include/usr/intr/interrupt.H
@@ -33,13 +33,25 @@ namespace INTR
 
     /**
      * External Interrupt Types (XISR)
+     * This value is passed in message data[0] on interrupt or shutdown.
+     * @note The XISR is 24 bits:
+     * XISR[ 0: 4] NOT DEFINED
+     * XISR[ 5: 7] Node id
+     * XISR[ 8:10] chipId within Node 
+     * XISR[11:12] Unit selection [GX='00', PHB0='01', PHB1='10', PHB2='11']
+     * XISR[13:19] BUID
+     * XISR[20:23] level
+     *
+     * extr_intr_t values are XISR right shifted, lsb = bit 23
      */
     enum ext_intr_t
     {
         NO_INTERRUPT = 0,       //!< no interrupt present
-        INTERPROC = 2,          //!< Inter processor interrupt
+        INTERPROC = 2,          //!< Inter processor interrupt (should be changed?)
         FSP_MAILBOX = 0x1A,     //!< TODO final value??
         ATTENTION = 0x26,       //!< TODO find this value
+        MAX_XISR  = 0x00FFFFFF, //!< Max value of the XISR
+        SHUT_DOWN = 0x01000000, //!< INTR presenter sends this when shutting down
     };
 
     /**
@@ -57,12 +69,29 @@ namespace INTR
 
 
     /**
-     * Register a message queue to recieve external interrupts
-     * @param[in] i_msgQ  The message queue
-     * @param[in] i_type  The type of interrupt (XISR value)
-     * @return errlHndl_t on error.
+     * Register a message queue for an interrupt type
+     * @param[in] i_msgQ The message queue
+     * @param[in] i_msg_type, The message type of the message to send
+     *                        to i_msgQ when an interrupt of
+     *                        i_intr_type occurrs.
+     * @param[in] i_intr_type, The interrupt type to register.
+     *
+     * @note the interrupt type is currently the XISR value in the XIRR
+     *       register and consists of the chipid, buid, and level
+     *       @see  i_intr_type for enumerations.
+     *
+     * @note when an interrupt of type i_msg_type occurrs, the
+     * interrupt presenter sends a sync message with type i_msg_type to
+     * i_msgQ with i_intr_type in message data word 0 and then waits
+     * for a response.
+     *
+     * @note When HB is shutting down the interrupt presenter will send
+     * a message to all registered queues with a sync message type of
+     * i_msg_type, and data[0] = INTR::SHUT_DOWN.  A response is expected.
      */
-    errlHndl_t registerMsgQ(msg_q_t i_msgQ, ext_intr_t i_type);
+    errlHndl_t registerMsgQ(msg_q_t i_msgQ,
+                            uint32_t i_msg_type,
+                            ext_intr_t i_intr_type);
 
     /**
      * Un register a message queue from the interrupt handler
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index ce2e1a26f..2683ca43b 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -93,12 +93,6 @@ errlHndl_t IntrRp::_init()
     // Make master thread 0
     uint32_t cpuid = task_getcpuid();
     iv_masterCpu = cpuid;
-    // If P7 or P7+ core -- need to tweak fields in PIR
-    if(cpu_core_type() == CORE_POWER7 ||
-       cpu_core_type() == CORE_POWER7_PLUS)
-    {
-        iv_masterCpu = P7PIR_t(cpuid);
-    }
     iv_masterCpu.threadId = 0;
 
     TRACFCOMP(g_trac_intr,"Master cpu node[%d], chip[%d], core[%d], thread[%d]",
@@ -245,16 +239,21 @@ void IntrRp::msgHandler()
                     Registry_t::iterator r = iv_registry.find(type);
                     if(r != iv_registry.end())
                     {
-                        msg_q_t msgQ = r->second;
+                        msg_q_t msgQ = r->second.msgQ;
+
                         msg_t * rmsg = msg_allocate();
-                        rmsg->type = type;
+                        rmsg->type = r->second.msgType;
+                        rmsg->data[0] = type;  // interrupt type
+                        rmsg->data[1] = 0;
+                        rmsg->extra_data = NULL;
+
                         int rc = msg_sendrecv(msgQ,rmsg);
                         if(rc)
                         {
                             TRACFCOMP(g_trac_intr,ERR_MRK
                                       "External Interrupt recieved type = %d, "
                                       "but could not send message to registered"
-                                      " handler. Ignorming it. rc = %d",
+                                      " handler. Ignoring it. rc = %d",
                                       (uint32_t) type, rc);
                         }
                         msg_free(rmsg);
@@ -278,8 +277,9 @@ void IntrRp::msgHandler()
             case MSG_INTR_REGISTER_MSGQ:
                 {
                     msg_q_t l_msgQ = reinterpret_cast<msg_q_t>(msg->data[0]);
-                    ext_intr_t l_t = static_cast<ext_intr_t>(msg->data[1]);
-                    errlHndl_t err = registerInterrupt(l_msgQ,l_t);
+                    uint64_t l_type = msg->data[1];
+                    ext_intr_t l_intr_type = static_cast<ext_intr_t>(l_type & 0xFFFF);
+                    errlHndl_t err = registerInterrupt(l_msgQ,l_type >> 32,l_intr_type);
 
                     msg->data[1] = reinterpret_cast<uint64_t>(err);
                     msg_respond(iv_msgQ,msg);
@@ -297,7 +297,7 @@ void IntrRp::msgHandler()
                     Registry_t::iterator r = iv_registry.find(type);
                     if(r != iv_registry.end())
                     {
-                        msgQ = r->second;
+                        msgQ = r->second.msgQ;
                         iv_registry.erase(r);
                     }
 
@@ -333,12 +333,8 @@ void IntrRp::msgHandler()
             case MSG_INTR_ADD_CPU:
                 {
                     PIR_t pir = msg->data[0];
-                    // If P7 or P7+ core -- need to tweak fields in PIR
-                    if(cpu_core_type() == CORE_POWER7 ||
-                       cpu_core_type() == CORE_POWER7_PLUS)
-                     {
-                        pir = P7PIR_t(msg->data[0]);
-                    }
+                    pir.threadId = 0;
+                    iv_cpuList.push_back(pir);
 
                     TRACFCOMP(g_trac_intr,"Add CPU node[%d], chip[%d],"
                               "core[%d], thread[%d]",
@@ -359,38 +355,13 @@ void IntrRp::msgHandler()
                 }
                 break;
 
-            case MSG_INTR_REMOVE_CPU:
-                {
-                    PIR_t pir = msg->data[0];
-                    if(cpu_core_type() == CORE_POWER7 ||
-                       cpu_core_type() == CORE_POWER7_PLUS)
-                    {
-                        pir = P7PIR_t(msg->data[0]);
-                    }
-
-                    TRACFCOMP(g_trac_intr,"Remove CPU node[%d], chip[%d],"
-                              "core[%d], thread[%d]",
-                              pir.nodeId, pir.chipId, pir.coreId,
-                              pir.threadId);
-
-                    size_t threads = cpu_thread_count();
-
-                    for(size_t thread = 0; thread < threads; ++thread)
-                    {
-                        pir.threadId = thread;
-                        deconfigureInterruptPresenter(pir);
-                    }
-
-                    msg->data[1] = 0;
-                    msg_respond(iv_msgQ, msg);
-                }
-                break;
-
             case MSG_INTR_SHUTDOWN:
                 {
                     TRACFCOMP(g_trac_intr,"Shutdown event received");
-                    // TODO rtc story 39878 for content
+                    shutDown();
+
                     msg_respond(iv_msgQ, msg);
+
                 }
                 break;
 
@@ -432,18 +403,21 @@ errlHndl_t IntrRp::setBAR(TARGETING::Target * i_target,
 
 
 
-errlHndl_t IntrRp::registerInterrupt(msg_q_t i_msgQ, ext_intr_t i_type)
+errlHndl_t IntrRp::registerInterrupt(msg_q_t i_msgQ,
+                                     uint32_t i_msg_type,
+                                     ext_intr_t i_intr_type)
 {
     errlHndl_t err = NULL;
 
-    Registry_t::iterator r = iv_registry.find(i_type);
+    Registry_t::iterator r = iv_registry.find(i_intr_type);
     if(r == iv_registry.end())
     {
-        iv_registry[i_type] = i_msgQ;
+        TRACDCOMP(g_trac_intr,"INTR::register %x", i_intr_type);
+        iv_registry[i_intr_type] = intr_response_t(i_msgQ,i_msg_type);
     }
     else
     {
-        if(r->second != i_msgQ)
+        if(r->second.msgQ != i_msgQ)
         {
         /*@ errorlog tag
          * @errortype       ERRL_SEV_INFORMATIONAL
@@ -460,7 +434,7 @@ errlHndl_t IntrRp::registerInterrupt(msg_q_t i_msgQ, ext_intr_t i_type)
                  ERRORLOG::ERRL_SEV_INFORMATIONAL,    // severity
                  INTR::MOD_INTRRP_REGISTERINTERRUPT,  // moduleid
                  INTR::RC_ALREADY_REGISTERED,         // reason code
-                 i_type,
+                 i_intr_type,
                  0
                 );
         }
@@ -580,8 +554,60 @@ errlHndl_t IntrRp::checkAddress(uint64_t i_addr)
     return err;
 }
 
+void IntrRp::shutDown()
+{
+    msg_t * rmsg = msg_allocate();
+
+    // Call everyone and say shutting down!
+    for(Registry_t::iterator r = iv_registry.begin();
+        r != iv_registry.end();
+        ++r)
+    {
+        msg_q_t msgQ = r->second.msgQ;
+
+        rmsg->type = r->second.msgType;
+        rmsg->data[0] = SHUT_DOWN;
+        rmsg->data[1] = 0;
+        rmsg->extra_data = NULL;
+
+        int rc = msg_sendrecv(msgQ,rmsg);
+        if(rc)
+        {
+            TRACFCOMP(g_trac_intr,ERR_MRK
+                      "Could not send message to registered handler to Shut"
+                      " down. Ignoring it.  rc = %d",
+                      rc);
+        }
+    }
+
+    msg_free(rmsg);
+
+    // Reset the hardware regiseters
+
+    iv_cpuList.push_back(iv_masterCpu);
+
+    size_t threads = cpu_thread_count();
+    for(CpuList_t::iterator pir_itr = iv_cpuList.begin();
+        pir_itr != iv_cpuList.end();
+        ++pir_itr)
+    {
+        PIR_t pir = *pir_itr;
+        for(size_t thread = 0; thread < threads; ++thread)
+        {
+            pir.threadId = thread;
+            deconfigureInterruptPresenter(pir);
+        }
+    }
+}
+
+//----------------------------------------------------------------------------
+// External interfaces
+//----------------------------------------------------------------------------
+
 // Register a message queue with a particular intr type
-errlHndl_t INTR::registerMsgQ(msg_q_t i_msgQ, ext_intr_t i_type)
+errlHndl_t INTR::registerMsgQ(msg_q_t i_msgQ,
+                              uint32_t i_msg_type,
+                              ext_intr_t i_intr_type)
 {
     errlHndl_t err = NULL;
     // Can't add while handling an interrupt, so
@@ -592,7 +618,8 @@ errlHndl_t INTR::registerMsgQ(msg_q_t i_msgQ, ext_intr_t i_type)
         msg_t * msg = msg_allocate();
         msg->type = MSG_INTR_REGISTER_MSGQ;
         msg->data[0] = reinterpret_cast<uint64_t>(i_msgQ);
-        msg->data[1] = static_cast<uint64_t>(i_type);
+        msg->data[1] = static_cast<uint64_t>(i_intr_type);
+        msg->data[1] |= static_cast<uint64_t>(i_msg_type) << 32;
 
         int rc = msg_sendrecv(intr_msgQ, msg);
         if(!rc)
@@ -626,7 +653,7 @@ errlHndl_t INTR::registerMsgQ(msg_q_t i_msgQ, ext_intr_t i_type)
              ERRORLOG::ERRL_SEV_INFORMATIONAL,    // severity
              INTR::MOD_INTR_REGISTER,             // moduleid
              INTR::RC_REGISTRY_NOT_READY,         // reason code
-             static_cast<uint64_t>(i_type),
+             static_cast<uint64_t>(i_intr_type),
              0
             );
     }
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 646d0761d..95c510603 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -123,29 +123,6 @@ namespace INTR
             };
 
             /**
-             * cpu P7PIR register
-             * @note      P7 bits - thread 2, core 3, chip 2, node 3,
-             */
-            struct P7PIR_t
-            {
-                union
-                {
-                    uint32_t word;
-                    struct
-                    {
-                        uint32_t        reserved:22;    //!< zeros
-                        uint32_t        nodeId:3;       //!< node (8)
-                        uint32_t        chipId:2;       //!< chip pos on node (4)
-                        uint32_t        coreId:3;       //!< Core number (8)
-                        uint32_t        threadId:2;     //!< thread number (4)
-                    } PACKED;
-                };
-                P7PIR_t(uint32_t i_word) : word(i_word) {}
-            };
-
-
-
-            /**
              * cpu PIR register
              */
             struct PIR_t
@@ -170,26 +147,37 @@ namespace INTR
                     word = i_word;
                     return word;
                 }
+            };
 
-                PIR_t operator = (P7PIR_t i_p7pir)
-                {
-                    nodeId      = i_p7pir.nodeId;
-                    chipId      = i_p7pir.chipId;
-                    coreId      = i_p7pir.coreId;
-                    threadId    = i_p7pir.threadId;
-
-                    return word;
-                }
+            struct intr_response_t
+            {
+                msg_q_t msgQ;
+                uint32_t msgType;
+
+                /**
+                 * Default Constructor
+                 */
+                intr_response_t(): msgQ(NULL), msgType(0) {}
+
+                /**
+                 * Constructor
+                 * @param[in] i_msgQ, The message queue
+                 * @param[in] i_msgType, The message type
+                 */
+                intr_response_t(msg_q_t i_msgQ, uint32_t i_msgType) :
+                    msgQ(i_msgQ), msgType(i_msgType) {}
             };
 
 
-            typedef std::map<ext_intr_t,msg_q_t> Registry_t;
+            typedef std::map<ext_intr_t,intr_response_t> Registry_t;
+            typedef std::vector<PIR_t> CpuList_t;
 
 
             msg_q_t     iv_msgQ;        //!< Kernel Interrupt message queue
             Registry_t  iv_registry;    //!< registered interrupt type
             uint64_t    iv_baseAddr;    //!< Base address of hw INTR regs
             PIR_t       iv_masterCpu;   //!< Master cpu PIR
+            CpuList_t   iv_cpuList;     //!< Other CPU chips
 
 
         private:        //functions
@@ -204,9 +192,24 @@ namespace INTR
             /**
              * Register a message queue for an interrupt type
              * @param[in] i_msgQ The message queue
-             * @param[in] i_type the interrupt type
+             * @param[in] i_msg_type, The message type of the message to send
+             *                        to i_msgQ when an interrupt of
+             *                        i_intr_type occurrs.
+             * @param[in] i_intr_type, The interrupt type to register.
+             *
+             * @note the interrupt type is currently the XISR value in the XIRR
+             *       register and consists of the chipid, buid, and level
+             *       @see src/include/usr/intr/interrupt.H i_intr_type for
+             *       enumerations.
+             *
+             * @note when an interrupt of type i_msg_type occurrs, the
+             * interrupt presenter sends a message with type i_msg_type to
+             * i_msgQ with i_intr_type in message data word 0 and then waits
+             * for a response.
              */
-            errlHndl_t registerInterrupt(msg_q_t i_msgQ, ext_intr_t i_type);
+            errlHndl_t registerInterrupt(msg_q_t i_msgQ, 
+                                         uint32_t i_msg_type,
+                                         ext_intr_t i_intr_type);
 
             /**
              * Enable hardware to reporting external interrupts
@@ -239,6 +242,11 @@ namespace INTR
                               const PIR_t i_pir);
 
             /**
+             * Shutdown procedure
+             */
+            void shutDown();
+
+            /**
              * Calculate the adress offset for the given cpu
              * @param[in] i_pir PIR value for the presenter
              * @return the offset
diff --git a/src/usr/intr/test/intrtest.H b/src/usr/intr/test/intrtest.H
index d395ab9ac..7cdfc0d0b 100644
--- a/src/usr/intr/test/intrtest.H
+++ b/src/usr/intr/test/intrtest.H
@@ -139,7 +139,7 @@ class IntrTest: public CxxTest::TestSuite
 
             // Need to register a msgq
             msg_q_t msgQ = msg_q_create();
-            err = INTR::registerMsgQ(msgQ,INTR::INTERPROC);
+            err = INTR::registerMsgQ(msgQ,0,INTR::INTERPROC);
             if(err)
             {
                 TS_FAIL("Errl from INTR::registerMsgQ()");
@@ -154,10 +154,10 @@ class IntrTest: public CxxTest::TestSuite
             *(mfrr) = 0xff;
 
             msg_t* msg = msg_wait(msgQ); // wait for interrupt msg
-            TRACFCOMP(g_trac_intr,"Interrupt handled! Type=%lx",msg->type);
-            if(msg->type != INTR::INTERPROC)
+            TRACFCOMP(g_trac_intr,"Interrupt handled! Type=%lx",msg->data[0]);
+            if(msg->data[0] != INTR::INTERPROC)
             {
-                TS_FAIL("INTR::unexpected interrupt type %lx",msg->type);
+                TS_FAIL("INTR::unexpected interrupt type %lx",msg->data[0]);
             }
             msg_respond(msgQ,msg);
 
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index e8b017a70..e687a0a0e 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -92,7 +92,7 @@ errlHndl_t MailboxSp::_init()
     rc = msg_q_register(iv_msgQ, VFS_ROOT_MSG_MBOX);
 
     // Register to get interrupts for mailbox
-    err = INTR::registerMsgQ(iv_msgQ, INTR::FSP_MAILBOX);
+    err = INTR::registerMsgQ(iv_msgQ, MSG_INTR, INTR::FSP_MAILBOX);
 
     if(!err)
     {
@@ -147,159 +147,18 @@ void MailboxSp::msgHandler()
             // Interrupt from the mailbox hardware
             case MSG_INTR:
                 {
-                    mbox_msg_t mbox_msg;
-                    size_t mbox_msg_size = sizeof(mbox_msg_t);
-                    uint64_t mbox_status = 0;
-
-                    // Read message from DD
-                    err = DeviceFW::deviceRead(iv_trgt,
-                                               static_cast<void*>(&mbox_msg),
-                                               mbox_msg_size,
-                                               DEVICE_MBOX_ADDRESS(&mbox_status)
-                                              );
-                    if(err)
+                    if(msg->data[0] != INTR::SHUT_DOWN)
                     {
-                        err->collectTrace(HBMBOX_TRACE_NAME);
-                        err->collectTrace(HBMBOXMSG_TRACE_NAME);
-                        // still need to repond to interrupt handler
-                        // will cause assert() below.
+                        err = handleInterrupt();
                     }
                     else
                     {
-
-                        TRACDCOMP(g_trac_mbox,"MBOXSP status=%lx",mbox_status);
-
-                        if(mbox_status & MBOX_HW_ACK)
-                        {
-                            // send next message if there is one.
-                            iv_rts = true;
-                            send_msg();
-                        }
-
-                        if(mbox_status & MBOX_DATA_PENDING)
-                        {
-                            trace_msg("RECV",mbox_msg);
-                            if(mbox_msg.msg_queue_id == HB_MAILBOX_MSGQ)
-                            {
-                                // msg to hb mailbox from fsp mbox
-                                handle_hbmbox_msg(mbox_msg);
-                            }
-                            else if((mbox_msg.msg_queue_id==FSP_MAILBOX_MSGQ)&&
-                                    (mbox_msg.msg_payload.type ==
-                                     MSG_REQUEST_DMA_BUFFERS))
-                            {
-                                // This is a response from The FSP
-                                handle_hbmbox_resp(mbox_msg);
-                            }
-                            else
-                            {
-                                // anything else
-                                recv_msg(mbox_msg);
-                            }
-                        }
-
-                        // Look for error status from MB hardware
-                        if(mbox_status & MBOX_DOORBELL_ERROR)
-                        {
-                            TRACFCOMP(g_trac_mbox,
-                                      ERR_MRK"MBOX status 0x%lx",
-                                      mbox_status);
-
-                            if(mbox_status & MBOX_DATA_WRITE_ERR)
-                            {
-                                // Write attempted before ACK - HB code error
-                                /*@ errorlog tag
-                                 * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
-                                 * @moduleid        MOD_MBOXSRV_HNDLR
-                                 * @reasoncode      RC_DATA_WRITE_ERR
-                                 * @userdata1       Status from MB device driver
-                                 * @defdesc         Mailbox Data Write attempted
-                                 *                  before ACK.
-                                 *
-                                 */
-                                err = new ERRORLOG::ErrlEntry
-                                    (
-                                     ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
-                                     MBOX::MOD_MBOXSRV_HNDLR,
-                                     MBOX::RC_DATA_WRITE_ERR,  //  reason Code
-                                     mbox_status,              //  Status from DD
-                                     0
-                                    );
-
-                                err->collectTrace(HBMBOX_TRACE_NAME);
-                                err->collectTrace(HBMBOXMSG_TRACE_NAME);
-                                // will assert later below
-                            }
-
-                            else if(mbox_status & MBOX_PARITY_ERR)
-                            {
-                                // Hardware detected parity error
-                                // - TODO How does BB handle this error ???
-                                // Log it and continue
-
-                                /*@ errorlog tag
-                                 * @errortype       ERRL_SEV_INFORMATIONAL
-                                 * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
-                                 * @reasoncode      MBOX::RC_PARITY_ERR
-                                 * @userdata1       Status from MB device driver
-                                 * @defdesc         Mailbox Hardware detected
-                                 *                  parity error.
-                                 */
-                                err = new ERRORLOG::ErrlEntry
-                                    (
-                                     ERRORLOG::ERRL_SEV_INFORMATIONAL,
-                                     MBOX::MOD_MBOXSRV_HNDLR,
-                                     MBOX::RC_PARITY_ERR,    //  reason Code
-                                     mbox_status,            //  Status from DD
-                                     0
-                                    );
-
-                                err->collectTrace(HBMBOX_TRACE_NAME);
-                                errlCommit(err,HBMBOX_COMP_ID);
-                                // err = NULL
-                            }
-                            else if(mbox_status & MBOX_ILLEGAL_OP)
-                            {
-                                // Code problem could be FSP or HB 
-                                // - log error and continue.
-
-                                /*@ errorlog tag
-                                 * @errortype       ERRL_SEV_INFORMATIONAL
-                                 * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
-                                 * @reasoncode      MBOX::RC_ILLEGAL_OP
-                                 * @userdata1       Status from MB device driver
-                                 * @defdesc         Retry failed. Bad status
-                                 *                  indicated in PIB status reg.
-                                 *
-                                 */
-                                err = new ERRORLOG::ErrlEntry
-                                    (
-                                     ERRORLOG::ERRL_SEV_INFORMATIONAL,
-                                     MBOX::MOD_MBOXSRV_HNDLR,
-                                     MBOX::RC_ILLEGAL_OP,    //  reason Code
-                                     mbox_status,            //  Status from DD
-                                     0
-                                    );
-                                err->collectTrace(HBMBOX_TRACE_NAME);
-                                errlCommit(err,HBMBOX_COMP_ID);
-                                // err = NULL
-                            }
-
-                            //else if(mbox_status & MBOX_DATA_READ_ERR)
-                            //{
-                            //    // Read when no message available 
-                            //    - Just ignore this one per Dean.
-                            //}
-
-
-                            // The code has the ability to retry sending a
-                            // message, but currently there is no error state
-                            // that requires it.
-                        }
+                        // Shutdown the mailbox
+                        TRACFCOMP(g_trac_mbox,"MBOXSP Shutdown event received");
+                        // TODO in story 41136
                     }
 
-
-                    // Respond to the interrupt handler
+                    // Respond to the interrupt handler regardless of err
                     msg->data[0] = 0;
                     msg->data[1] = 0;
                     msg_respond(iv_msgQ,msg);
@@ -308,7 +167,7 @@ void MailboxSp::msgHandler()
                     // or MBOX_DATA_WRITE_ERR  - serious - assert
                     if(err)
                     {
-                        // TODO Mask off MB interrupts??
+                        // TODO  story 37990
                         errlCommit(err,HBMBOX_COMP_ID);
 
                         assert(0);
@@ -1034,6 +893,163 @@ void MailboxSp::trace_msg(const char * i_text,
               i_mbox_msg.msg_payload.extra_data);
 }
 
+
+errlHndl_t MailboxSp::handleInterrupt()
+{
+    errlHndl_t err = NULL;
+    mbox_msg_t mbox_msg;
+    size_t mbox_msg_size = sizeof(mbox_msg_t);
+    uint64_t mbox_status = 0;
+
+    // Read message from DD
+    err = DeviceFW::deviceRead(iv_trgt,
+                               static_cast<void*>(&mbox_msg),
+                               mbox_msg_size,
+                               DEVICE_MBOX_ADDRESS(&mbox_status)
+                              );
+    if(err)
+    {
+        err->collectTrace(HBMBOX_TRACE_NAME);
+        err->collectTrace(HBMBOXMSG_TRACE_NAME);
+
+        // return err
+    }
+    else
+    {
+        TRACDCOMP(g_trac_mbox,"MBOXSP status=%lx",mbox_status);
+
+        if(mbox_status & MBOX_HW_ACK)
+        {
+            // send next message if there is one.
+            iv_rts = true;
+            send_msg();
+        }
+
+        if(mbox_status & MBOX_DATA_PENDING)
+        {
+            trace_msg("RECV",mbox_msg);
+            if(mbox_msg.msg_queue_id == HB_MAILBOX_MSGQ)
+            {
+                // msg to hb mailbox from fsp mbox
+                handle_hbmbox_msg(mbox_msg);
+            }
+            else if((mbox_msg.msg_queue_id==FSP_MAILBOX_MSGQ)&&
+                    (mbox_msg.msg_payload.type == MSG_REQUEST_DMA_BUFFERS))
+            {
+                // This is a response from the FSP
+                handle_hbmbox_resp(mbox_msg);
+            }
+            else
+            {
+                // anything else
+                recv_msg(mbox_msg);
+            }
+        }
+
+        // Look for error status from MB hardware
+        if(mbox_status & MBOX_DOORBELL_ERROR)
+        {
+            TRACFCOMP(g_trac_mbox,
+                      ERR_MRK"MBOX status 0x%lx",
+                      mbox_status);
+
+            if(mbox_status & MBOX_DATA_WRITE_ERR)
+            {
+                // Write attempted before ACK - HB code error
+                /*@ errorlog tag
+                 * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+                 * @moduleid        MOD_MBOXSRV_HNDLR
+                 * @reasoncode      RC_DATA_WRITE_ERR
+                 * @userdata1       Status from MB device driver
+                 * @defdesc         Mailbox Data Write attempted
+                 *                  before ACK.
+                 *
+                 */
+                err = new ERRORLOG::ErrlEntry
+                    (
+                     ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
+                     MBOX::MOD_MBOXSRV_HNDLR,
+                     MBOX::RC_DATA_WRITE_ERR,  //  reason Code
+                     mbox_status,              //  Status from DD
+                     0
+                    );
+
+                err->collectTrace(HBMBOX_TRACE_NAME);
+                err->collectTrace(HBMBOXMSG_TRACE_NAME);
+                // return err
+            }
+
+            else if(mbox_status & MBOX_PARITY_ERR)
+            {
+                // Hardware detected parity error
+                // - TODO How does BB handle this error ???
+                // Log it and continue
+
+                /*@ errorlog tag
+                 * @errortype       ERRL_SEV_INFORMATIONAL
+                 * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
+                 * @reasoncode      MBOX::RC_PARITY_ERR
+                 * @userdata1       Status from MB device driver
+                 * @defdesc         Mailbox Hardware detected
+                 *                  parity error.
+                 */
+                err = new ERRORLOG::ErrlEntry
+                    (
+                     ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                     MBOX::MOD_MBOXSRV_HNDLR,
+                     MBOX::RC_PARITY_ERR,    //  reason Code
+                     mbox_status,            //  Status from DD
+                     0
+                    );
+
+                err->collectTrace(HBMBOX_TRACE_NAME);
+                errlCommit(err,HBMBOX_COMP_ID);
+                // err = NULL don't return err.
+            }
+            else if(mbox_status & MBOX_ILLEGAL_OP)
+            {
+                // Code problem could be FSP or HB 
+                // - log error and continue.
+
+                /*@ errorlog tag
+                 * @errortype       ERRL_SEV_INFORMATIONAL
+                 * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
+                 * @reasoncode      MBOX::RC_ILLEGAL_OP
+                 * @userdata1       Status from MB device driver
+                 * @defdesc         Retry failed. Bad status
+                 *                  indicated in PIB status reg.
+                 *
+                 */
+                err = new ERRORLOG::ErrlEntry
+                    (
+                     ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                     MBOX::MOD_MBOXSRV_HNDLR,
+                     MBOX::RC_ILLEGAL_OP,    //  reason Code
+                     mbox_status,            //  Status from DD
+                     0
+                    );
+                err->collectTrace(HBMBOX_TRACE_NAME);
+                errlCommit(err,HBMBOX_COMP_ID);
+                // err = NULL Don't return err
+            }
+
+            //else if(mbox_status & MBOX_DATA_READ_ERR)
+            //{
+            //    // Read when no message available 
+            //    - Just ignore this one per Dean.
+            //}
+
+
+            // The code has the ability to retry sending a
+            // message, but currently there is no error state
+            // that requires it.
+        }
+    }
+
+    return err;
+}
+
+
 // ----------------------------------------------------------------------------
 // External Interfaces @see mboxif.H
 // ----------------------------------------------------------------------------
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index b3c806d28..0980940c2 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -55,11 +55,7 @@ namespace MBOX
         MSG_SEND,
         MSG_REGISTER_MSGQ,
         MSG_UNREGISTER_MSGQ,
-
-        // TODO currently this is progammed to 0x1A in simics, but could change.
-        // Fix this when intr handler is changed to return this as data[0]
-        // instead of type.
-        MSG_INTR = INTR::FSP_MAILBOX,   // Currently defined in intr/interupt.H
+        MSG_INTR,
     };
 
 
@@ -172,6 +168,12 @@ namespace MBOX
             msg_q_t msgq_unregister(queue_id_t i_queue_id);
 
             /**
+             * Handle interrupt from Intr presenter
+             * @param[in] i_msg, The message 
+             */
+            errlHndl_t handleInterrupt();
+
+            /**
              * Trace the message to the "fast" trace buffer
              * @param[in] i_text, a Description
              * @param[in] i_mbox_msg, the message to trace
diff --git a/src/usr/mbox/test/mboxddtest.H b/src/usr/mbox/test/mboxddtest.H
index d60eb5e68..fa8a683bf 100644
--- a/src/usr/mbox/test/mboxddtest.H
+++ b/src/usr/mbox/test/mboxddtest.H
@@ -469,7 +469,7 @@ class MboxDDTest : public CxxTest::TestSuite
 
         /*Register for interrupt*/
         msg_q_t msgQ = msg_q_create();
-        l_err = INTR::registerMsgQ(msgQ,(INTR::ext_intr_t)0x1A);    //Simics is setup to send IRQ on 26
+        l_err = INTR::registerMsgQ(msgQ,0,(INTR::ext_intr_t)0x1A);    //Simics is setup to send IRQ on 26
         if(l_err)
             {
             TS_FAIL("Errl from INTR::registerMsgQ()");
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index 161f7d446..080f21ccf 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -88,7 +88,7 @@ class MboxSPTest : public CxxTest::TestSuite
 
           // Send last message
           msg_t * msg = msg_allocate();
-          msg->type = 3;        // use this to terminate while loop below
+          msg->type = 0xff;        // use this to terminate while loop below
           msg->extra_data = NULL;
           err = MBOX::send(MBOX::HB_TEST_MSGQ,msg);
           if(err)
@@ -105,7 +105,7 @@ class MboxSPTest : public CxxTest::TestSuite
           {
               msg_t* msg = msg_wait(msgQ);
 
-              if(msg->type == 3)
+              if(msg->type == 0xff)
               {
                   msg_free(msg);
                   break;