Multi-Drawer (IPC) Interrupt/Messaging Support

 - Use doorbells instead of IPIs (no IPI support
   using LSI interupts in the XIVE intr architecture)

 - New message type from kernel to userspace so the
   kernel can notify the HB userspace Interrupt
   Resource Provider (INTRP) that an IPC message was
   sent to the particular HB instance (in P8 this
   happened automatically as that was part of the
   IPI architecture).

 - Re-enable testcase that validates that an IPC
   message can be successfully sent.

Change-Id: Ic846f8dca45217205ed61d8381a573e995cb16f2
RTC: 150861
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/52004
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

17 files changed, 222 insertions, 195 deletions

diff --git a/src/include/kernel/doorbell.H b/src/include/kernel/doorbell.H
index 8275c4fc7..1d01ce119 100644
--- a/src/include/kernel/doorbell.H
+++ b/src/include/kernel/doorbell.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -50,11 +50,11 @@ void doorbell_send(uint64_t i_pir);
  */
 void send_doorbell_wakeup(uint64_t i_pir);
 
-/* TODO RTC 150861
- * send_doorbell_ipc
+/** Send an IPC message using a doorbell
  *
- * void send_doorbell_ipc(cpu_t *i_cpu, uint64_t pir);
-**/
+ * @param i_pir - PIR to send IPC message to
+ */
+void send_doorbell_ipc(uint64_t i_pir);
 
 enum
 {
diff --git a/src/include/kernel/intmsghandler.H b/src/include/kernel/intmsghandler.H
index 8f7c08559..9e5a001ed 100644
--- a/src/include/kernel/intmsghandler.H
+++ b/src/include/kernel/intmsghandler.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -73,8 +73,11 @@ class InterruptMsgHdlr : public MessageHandler
             ACK_HYPERVISOR_INT_REG_OFFSET = 0x830,
 
             INTP_BAR_VALUE     = 0xFFFFE000,    // upper 32 bits of IPCBAR
-
             INTERPROC_XISR     = 2,     //IPI XISR is 2
+
+            MSG_KEY_THREAD_WKUP  = 0x8000000000000000ul,
+            MSG_KEY_ADD_CPU_CORE = 0x4000000000000000ul,
+            MSG_KEY_IPC_MSG      = 0x2000000000000000ul,
         };
 
         /**
@@ -156,6 +159,15 @@ class InterruptMsgHdlr : public MessageHandler
         static void sendThreadWakeupMsg(uint64_t i_pir);
 
         /**
+         * Send message to interrupt resource provider (intrrP) in userspace to
+         * indicate it was sent an IPC message. The intrrp will re-route the
+         * message to the appropriate handler.
+         *
+         * @param[in] i_pir - The PIR of the CPU to send doorbell to.
+         */
+        static void sendIpcMsg(uint64_t i_pir);
+
+        /**
          * Issue the sbe/mailbox workaround (issue a mbox EOI to mailbox)
          *
          */
diff --git a/src/include/kernel/workitem.H b/src/include/kernel/workitem.H
index 3a41cc526..6d2f4b6b1 100644
--- a/src/include/kernel/workitem.H
+++ b/src/include/kernel/workitem.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -54,7 +54,6 @@ class CpuWakeupDoorbellWorkItem : public KernelWorkItem
 };
 
 
-/* TODO RTC 150861
 
 //A work item to be created/executed during an IPC scenario
 class IPCDoorbellWorkItem : public KernelWorkItem
@@ -62,7 +61,9 @@ class IPCDoorbellWorkItem : public KernelWorkItem
     public:
         //Implement operator() function
         void operator() (void);
+
+        //No data to clean up, use default destructor
+        ~IPCDoorbellWorkItem() = default;
 };
-**/
 #endif
 
diff --git a/src/include/sys/msg.h b/src/include/sys/msg.h
index b161a91e8..2b7f2a8f4 100644
--- a/src/include/sys/msg.h
+++ b/src/include/sys/msg.h
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -66,6 +66,8 @@ enum msg_sys_types_t
     MSG_INTR_CPU_WAKEUP, //!< Msg sent from kernel to user space on cpu wakeup
                          // data[0] = cpuid (PIR)
     MSG_INTR_ISSUE_SBE_MBOX_WA,   //!< Issue EOI to mailbox
+    MSG_INTR_IPC,       //!< Msg sent from kernel to user space for multi-node
+                        // communication
 
 };
 
diff --git a/src/include/usr/intr/interrupt.H b/src/include/usr/intr/interrupt.H
index 3f4d0e8dd..b03f08f82 100644
--- a/src/include/usr/intr/interrupt.H
+++ b/src/include/usr/intr/interrupt.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -35,30 +35,9 @@ namespace TARGETING
 
 namespace INTR
 {
-    /**
-     * External Interrupt Types (XISR)
-     * This value is passed in message data[0] on interrupt or shutdown.
-     * @note The XISR is 24 bits:
-     * XISR[ 0: 4] NOT DEFINED
-     * XISR[ 5: 7] Node id
-     * XISR[ 8:10] chipId within Node
-     * XISR[11:12] Unit selection [GX='00', PHB0='01', PHB1='10', PHB2='11']
-     * XISR[13:19] BUID
-     * XISR[20:23] level
-     *
-     * extr_intr_t values are XISR right shifted, lsb = bit 23
-     */
-    enum XISRvalue_t
-    {
-        NO_INTERRUPT = 0,       //!< no interrupt present
-        //TODO RTC 137564
-        INTERPROC_XISR = 0xFF,  //!< XISR value for IPIs
-        MAX_XISR  = 0x00FFFFFF, //!< Max value of the XISR
-        SHUT_DOWN = 0x01000000, //!< INTR presenter sends this when shutting down
-    };
-
     typedef uint32_t ext_intr_t;
 
+    // Interrupt Source Number (ISN) Definition
     enum ISNvalue_t
     {
         ISN_PSI         = 0,
@@ -68,8 +47,8 @@ namespace INTR
         ISN_LPC         = 3,
         ISN_LCL_ERR     = 4,
         ISN_HOST        = 5,
-        ISN_INTERPROC   = 0xF0, //"special" as it isn't part of PSIHB
-                                //Converts to INTERPROC_XISR when registered
+        ISN_INTERPROC   = 0xF0, // Interproc (IPC) Message Type
+        SHUT_DOWN = 0x01000000, //!< INTR presenter sends this when shutting down
     };
 
     //These values are HW defined values from the LSI Interrupts status register
@@ -91,14 +70,8 @@ namespace INTR
         LSI_DIO         = 12,
         LSI_PSU         = 13,
         LSI_LAST_SOURCE,
-
-//TODO RTC 137564
-//        ISN_INTERPROC   = 0xF0, //"special" as it isn't part of PSIHB
-//                                //Converts to INTERPROC_XISR when registered
-//                                //
     };
 
-
     /**
      * The XISR value is the logical OR of the Interrup Requestor Source
      * Number (IRSN) and the Interrrupt Source Number (ISN).
diff --git a/src/include/usr/intr/intr_reasoncodes.H b/src/include/usr/intr/intr_reasoncodes.H
index 4101e2380..62d8ad87a 100644
--- a/src/include/usr/intr/intr_reasoncodes.H
+++ b/src/include/usr/intr/intr_reasoncodes.H
@@ -35,7 +35,6 @@ namespace INTR
         MOD_INTR_ENABLE              = 0x01, /**< intrrp.C : INTR::enableExternalInterrupts */
         MOD_INTR_DISABLE             = 0x02, /**< intrrp.C : INTR::disableExternalInterrupts */
         MOD_INTR_REGISTER            = 0x03, /**< intrrp.C : INTR::registerMsgQ */
-        MOD_INTRRP_CHECKADDRESS      = 0x04, /**< intrrp.C : IntrRp::checkAddress */
         MOD_INTRRP_REGISTERINTERRUPT = 0x05, /**< intrrp.C : IntrRp::registerInterrupt */
         MOD_INTR_ENABLE_PSI_INTR     = 0x06, /**< intrrp.C : INTR::enablePsiIntr */
         MOD_INTR_INIT_XIVR           = 0x07, /**< intrrp.C : INTR::initXIVR */
@@ -50,6 +49,7 @@ namespace INTR
         MOD_INTRRP_HNDLPSUINTERRUPT  = 0x10, /**< intrrp.C : INTR::handlePsuInterrupt */
         MOD_INTRRP_RESETINTUNIT      = 0x11, /**< intrrp.C : IntrRp::resetIntUnit */
         MOD_INTRRP_XIVE_SENDEOI      = 0x12,
+        MOD_INTRRP_IPC               = 0x13,
     };
 
     enum IntrReasonCode
@@ -67,6 +67,7 @@ namespace INTR
         RC_PSU_DOORBELL_TIMEOUT       = INTR_COMP_ID | 0x0A,
         RC_XIVE_PBUS_QUIESCE_TIMEOUT  = INTR_COMP_ID | 0x0B,
         RC_MESSAGE_SEND_ERROR         = INTR_COMP_ID | 0x0C,
+        RC_IPC_DATA_INVALID           = INTR_COMP_ID | 0x0D,
     };
 };
 
diff --git a/src/include/usr/mbox/mbox_reasoncodes.H b/src/include/usr/mbox/mbox_reasoncodes.H
index 6b9e3d8a9..76c6c2bf7 100644
--- a/src/include/usr/mbox/mbox_reasoncodes.H
+++ b/src/include/usr/mbox/mbox_reasoncodes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2014                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -67,6 +67,7 @@ namespace MBOX
         RC_MAILBOX_DISABLED          = MBOX_COMP_ID | 0x10,
         RC_IPC_INVALID_NODE          = MBOX_COMP_ID | 0x11,
         RC_MSG_SEND_ERROR            = MBOX_COMP_ID | 0x12,
+        RC_IPC_INVALID_DATA          = MBOX_COMP_ID | 0x13,
     };
 
 
diff --git a/src/include/usr/mbox/mboxif.H b/src/include/usr/mbox/mboxif.H
index ae753429a..140cabf37 100644
--- a/src/include/usr/mbox/mboxif.H
+++ b/src/include/usr/mbox/mboxif.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
diff --git a/src/kernel/doorbell.C b/src/kernel/doorbell.C
index 4b99e7aee..010b7e078 100644
--- a/src/kernel/doorbell.C
+++ b/src/kernel/doorbell.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -79,17 +79,14 @@ void send_doorbell_wakeup(uint64_t i_pir)
     doorbell_send(i_pir);
 }
 
-/*
-TODO RTC 150861
 void send_doorbell_ipc(uint64_t i_pir)
 {
-    cpu_t *l_cpu = getCpu(i_pir)
+    cpu_t *l_cpu = CpuManager::getCpu(i_pir);
 
-    printk("send_doorbell_ipc to pir: %lx\n", i_pir);
+    printkd("send_doorbell_ipc to pir: %lx\n", i_pir);
     //Create WorkItem and put on the stack to be executed during doorbell
-    // execution (if needed, otherwise can likely delete and just send doorbell)
-    KernelWorkItem* l_work = new IpcDoorbellWorkItem();
+    //   execution
+    KernelWorkItem* l_work = new IPCDoorbellWorkItem();
     l_cpu->doorbell_actions.push(l_work);
-    doorbell_send(i_pir)
+    doorbell_send(i_pir);
 }
-**/
diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C
index ea055ceed..b4b5254a7 100644
--- a/src/kernel/intmsghandler.C
+++ b/src/kernel/intmsghandler.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -165,7 +165,7 @@ void InterruptMsgHdlr::addCpuCore(uint64_t i_pir)
     {
         // To avoid conflict with interrupts on thread i_pir, change the key
         // for the message to be an invalid PIR.
-        uint64_t pir_key = i_pir | 0x4000000000000000ul;
+        uint64_t pir_key = i_pir | MSG_KEY_ADD_CPU_CORE;
 
         cv_instance->iv_lock.lock();
         cv_instance->sendMessage(MSG_INTR_ADD_CPU,
@@ -180,7 +180,7 @@ void InterruptMsgHdlr::sendThreadWakeupMsg(uint64_t i_pir)
     {
         // To avoid conflict with interrupts on thread i_pir, change the key
         // for the message to be an invalid PIR.
-        uint64_t pir_key = i_pir | 0x8000000000000000ul;
+        uint64_t pir_key = i_pir | MSG_KEY_THREAD_WKUP;
 
         cv_instance->iv_lock.lock();
         cv_instance->sendMessage(MSG_INTR_CPU_WAKEUP,
@@ -189,6 +189,21 @@ void InterruptMsgHdlr::sendThreadWakeupMsg(uint64_t i_pir)
     }
 }
 
+void InterruptMsgHdlr::sendIpcMsg(uint64_t i_pir)
+{
+    if(cv_instance)
+    {
+        // To avoid conflict with interrupts on thread i_pir, change the key
+        // for the message to be an invalid PIR.
+        uint64_t pir_key = i_pir | MSG_KEY_IPC_MSG;
+
+        cv_instance->iv_lock.lock();
+        cv_instance->sendMessage(MSG_INTR_IPC,
+                                (void*)pir_key,(void *)i_pir,NULL);
+        cv_instance->iv_lock.unlock();
+    }
+}
+
 MessageHandler::HandleResult InterruptMsgHdlr::handleResponse
 (
  msg_sys_types_t i_type,
diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C
index e7a9ead9a..f7df4aacb 100644
--- a/src/kernel/ipc.C
+++ b/src/kernel/ipc.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -28,6 +28,7 @@
 #include <kernel/intmsghandler.H>
 #include <kernel/console.H>
 #include <errno.h>
+#include <kernel/doorbell.H>
 
 using namespace KernelIpc;
 
@@ -55,10 +56,10 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     // single node system with an alt-master could be designed such that the
     // alt-master were on a different logical (power bus numbering) node.
     // Since it's not in plan to use this IPC mechanism in a single node
-    // system, this case will ge ignored for now.
+    // system, this case will get ignored for now.
     uint64_t this_node = getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
     uint64_t hrmor_offset = getHRMOR()-(this_node*(ipc_data_area.hrmor_base));
-    uint64_t dest_node = (i_q >> 32) & 0x07; 
+    uint64_t dest_node = (i_q >> 32) & 0x07;
     uint64_t dest_hrmor = (ipc_data_area.hrmor_base*dest_node) + hrmor_offset;
 
     uint64_t dest_addr = reinterpret_cast<uint64_t>(&ipc_data_area);
@@ -68,7 +69,7 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     printkd("IPC Dest addr %lx Q_id:%lx\n",dest_addr,i_q);
 
     // pointer to the ipc_data_area in the destination node
-    ipc_data_area_t * p_dest = 
+    ipc_data_area_t * p_dest =
         reinterpret_cast<ipc_data_area_t*>(dest_addr);
 
     // get lock on IPC data area in other node
@@ -87,11 +88,8 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
 
     printkd("IPC send from PIR %lx to PIR %x\n",getPIR(),p_dest->pir);
 
-    /* TODO RTC 150861
-    // send IPI - use this_node + 10 as favor level of interrupt
-    //P8 Call: InterruptMsgHdlr::sendIPI(p_dest->pir,this_node + 0x10);
-    //P9 Call (likely): send_doorbell_ipc(p_dest->pir);
-    **/
+    // send doorbell to interrupt the other drawer
+    send_doorbell_ipc(p_dest->pir);
 
     // The message allocation is freed here to make msg_send for IPC
     // messages behave the same as non-IPC msg_send; that is, the message
diff --git a/src/kernel/workitem.C b/src/kernel/workitem.C
index a883cdb8f..a66908514 100644
--- a/src/kernel/workitem.C
+++ b/src/kernel/workitem.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -41,10 +41,12 @@ void CpuWakeupDoorbellWorkItem::operator() (void)
     return;
 }
 
-/*TODO RTC 150861
-void IpcDoorbellWorkItem::operator() (void)
+void IPCDoorbellWorkItem::operator() (void)
 {
-    //Decide what needs to be done when the Ipc Doorbell is received
-    // (if anything)
+    size_t pir = getPIR();
+    printk("IPC msg pir %ld done\n", pir);
+    //Send message tot he intrrp in userspace indicating it has a pending IPC
+    // message.
+    InterruptMsgHdlr::sendIpcMsg(pir);
+    return;
 }
-**/
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 69abb9102..a164544d8 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] Google Inc.                                                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index e961f6395..a076e0fb5 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -930,14 +930,18 @@ void IntrRp::msgHandler()
                     }
                 }
                 break;
-/*TODO RTC 150861 -- I think a new IPC message type needs to be defined.
- *                   And the code below should be executed when this new message
- *                   type is received. The Kernel will send this message to
- *                   here (this intrrp code) during doorbell wakeup.
+            case MSG_INTR_IPC:
+                {
+                    errlHndl_t l_err = NULL;
+                    uint64_t l_xirr_pir = msg->data[0];
+
+                    TRACFCOMP(g_trac_intr,INFO_MRK
+                              "IntrRp::msgHandler Doorbell IPC msg received"
+                                  " for %d", l_xirr_pir);
 
                     // Now handle any IPC messages
-                    // If something is registered for IPIs
-                    // and msg is ready, then handle
+                    // If something is registered for the IPC msg
+                    Registry_t::iterator r = iv_registry.find(ISN_INTERPROC);
                     if(r != iv_registry.end() &&
                        (KernelIpc::ipc_data_area.msg_queue_id !=
                        IPC_DATA_AREA_CLEAR) &&
@@ -948,7 +952,7 @@ void IntrRp::msgHandler()
 
                         msg_t * rmsg = msg_allocate();
                         rmsg->type = r->second.msgType;
-                        rmsg->data[0] = type;  // interrupt type
+                        rmsg->data[0] = ISN_INTERPROC;
                         rmsg->data[1] = l_xirr_pir;
                         rmsg->extra_data = NULL;
 
@@ -956,17 +960,57 @@ void IntrRp::msgHandler()
                         if(rc)
                         {
                             TRACFCOMP(g_trac_intr,ERR_MRK
-                                      "IPI Interrupt received, but could "
+                                      "IPC message received, but could "
                                       "not send message to the registered "
                                       "handler. Ignoring it. rc = %d",
                                       rc);
                         }
                     }
-**/
+                    else if(KernelIpc::ipc_data_area.msg_queue_id ==
+                              IPC_DATA_AREA_CLEAR ||
+                            KernelIpc::ipc_data_area.msg_queue_id ==
+                              IPC_DATA_AREA_LOCKED)
+                    {
+                        TRACFCOMP(g_trac_intr,ERR_MRK
+                                   "IPC message received but data area is in"
+                                   " an invalid state. msg_queue_id = 0x%lx",
+                                   KernelIpc::ipc_data_area.msg_queue_id);
+                        /*@ errorlog tag
+                         * @errortype       ERRL_SEV_PREDICTIVE
+                         * @moduleid        INTR::MOD_INTRRP_IPC
+                         * @reasoncode      INTR::RC_IPC_DATA_INVALID
+                         * @userdata1       IPC Data Area MSG Queue ID
+                         * @userdata2       PIR
+                         * @devdesc         Error encountered routing IPC
+                         *                  message
+                         */
+                        l_err = new ERRORLOG::ErrlEntry
+                            (
+                             ERRORLOG::ERRL_SEV_PREDICTIVE,    // severity
+                             INTR::MOD_INTRRP_IPC,             // moduleid
+                             INTR::RC_IPC_DATA_INVALID,        // reason code
+                             KernelIpc::ipc_data_area.msg_queue_id,
+                             l_xirr_pir
+                            );
+                    }
+                    else
+                    {
+                        TRACFCOMP(g_trac_intr,ERR_MRK
+                           "IPC Message received type but nothing registered "
+                           "to handle it. Ignoring it.");
+                    }
+
+                    if (l_err)
+                    {
+                        l_err->collectTrace(INTR_COMP_NAME, 256);
+                        errlCommit(l_err, INTR_COMP_ID);
+                    }
+                }
+                break;
             case MSG_INTR_EOI:
                 {
                     // Use standrard EOI (End of Interrupt) sequence
-                    if(msg->data[0] != INTERPROC_XISR)
+                    if(msg->data[0] != ISN_INTERPROC)
                     {
                         uint64_t intSource = msg->data[0];
                         PIR_t l_pir = msg->data[1];
@@ -1342,7 +1386,7 @@ void IntrRp::routeInterrupt(intr_hdlr_t* i_proc,
     // interrupt source
     Registry_t::iterator r = iv_registry.find(i_type);
 
-    if(r != iv_registry.end() && i_type != INTERPROC_XISR)
+    if(r != iv_registry.end() && i_type != ISN_INTERPROC)
     {
         msg_q_t msgQ = r->second.msgQ;
 
@@ -1928,47 +1972,6 @@ msg_q_t IntrRp::unregisterInterruptXISR(ext_intr_t i_xisr)
     return msgQ;
 }
 
-void IntrRp::sendIPI(const PIR_t i_pir) const
-{
-    uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(i_pir);
-    volatile uint8_t * mfrr =
-        reinterpret_cast<uint8_t*>(baseAddr + MFRR_OFFSET);
-
-    eieio(); sync();
-    MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE);
-    (*mfrr) = IPI_USR_PRIO;
-}
-
-
-errlHndl_t IntrRp::checkAddress(uint64_t i_addr)
-{
-    errlHndl_t err = NULL;
-
-    if(i_addr < VMM_VADDR_DEVICE_SEGMENT_FIRST)
-    {
-        /*@ errorlog tag
-         * @errortype       ERRL_SEV_INFORMATIONAL
-         * @moduleid        INTR::MOD_INTRRP_CHECKADDRESS
-         * @reasoncode      INTR::RC_BAD_VIRTUAL_IO_ADDRESS
-         * @userdata1       The bad virtual address
-         * @userdata2       0
-         *
-         * @devdesc         The virtual address is not a valid IO address
-         *
-         */
-        err = new ERRORLOG::ErrlEntry
-            (
-             ERRORLOG::ERRL_SEV_INFORMATIONAL,
-             INTR::MOD_INTRRP_CHECKADDRESS,
-             INTR::RC_BAD_VIRTUAL_IO_ADDRESS,
-             i_addr,
-             0
-            );
-    }
-
-    return err;
-}
-
 void IntrRp::shutDown(uint64_t i_status)
 {
     msg_t * rmsg = msg_allocate();
@@ -2671,7 +2674,7 @@ void IntrRp::drainMpIplInterrupts(TARGETING::TargetHandleList & i_cores)
                     volatile uint32_t xirr_rw = *xirrPtr;
 
                     //If IPI need to set mfrr to 0xFF
-                    if(INTERPROC_XISR == xirr)
+                    if(ISN_INTERPROC == xirr)
                     {
                         *mfrrPtr = 0xFF;
                     }
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index f2f3cd26e..ce32fb233 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -216,7 +216,6 @@ void IpcSp::msgHandler()
 
                 break;
 
-
             default:
 
                 TRACFCOMP( g_trac_ipc,
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 7260d129b..7d06c2e89 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] Google Inc.                                                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
@@ -189,13 +189,10 @@ errlHndl_t MailboxSp::_init()
         }
     }
 
-    // @todo RTC:126643
-#if (0)
     // Register for IPC messages
     err = INTR::registerMsgQ(iv_msgQ,
                              MSG_IPC,
                              INTR::ISN_INTERPROC);
-#endif
 
     if(mbxComm)
     {
@@ -268,7 +265,6 @@ void MailboxSp::msgHandler()
                     printkd("MSG_INTR type - handling interrupt\n");
                     err = handleInterrupt();
 
-
                     printkd("MSG_INTR type - sending EOI\n");
                     // Respond to the interrupt handler regardless of err
                     INTR::sendEOI(iv_msgQ,msg);
@@ -447,10 +443,11 @@ void MailboxSp::msgHandler()
 
             case MSG_IPC: // Look for Interprocessor Messages
                 {
-
                     uint64_t msg_q_id = KernelIpc::ipc_data_area.msg_queue_id;
                     if(msg_q_id == IPC_DATA_AREA_LOCKED)
                     {
+                        TRACFCOMP(g_trac_mbox, INFO_MRK
+                             "MBOXSP IPC data area locked");
                         // msg is being written, but not yet ready to handle
                         msg_q_id = IPC_DATA_AREA_CLEAR;
                     }
@@ -464,12 +461,39 @@ void MailboxSp::msgHandler()
                         isync();
                         *ipc_msg = KernelIpc::ipc_data_area.msg_payload;
                         lwsync();
-                        // EOI has already been sent by intrp. Simply clear
-                        // ipc area so another message can be sent
+                        // Clear ipc area so another message can be sent
                         KernelIpc::ipc_data_area.msg_queue_id =
                             IPC_DATA_AREA_CLEAR;
                         handleIPC(static_cast<queue_id_t>(msg_q_id), ipc_msg);
                     }
+                    else
+                    {
+                        TRACFCOMP(g_trac_mbox, ERR_MRK
+                                "MBOXSP invalid data found in IPC data area: "
+                                " %0xlx", msg_q_id);
+
+                        TRACFBIN(g_trac_mbox, "IPC Data Area:",
+                              &KernelIpc::ipc_data_area,
+                              sizeof(KernelIpc::ipc_data_area));
+
+                        /*@ errorlog tag
+                         * @errortype       ERRL_SEV_PREDICTIVE
+                         * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
+                         * @reasoncode      MBOX::RC_IPC_INVALID_DATA
+                         * @userdata1       IPC Data Area MSG Queue ID
+                         * @devdesc         IPC Message data corrupted
+                         */
+                        err = new ERRORLOG::ErrlEntry
+                            (
+                             ERRORLOG::ERRL_SEV_PREDICTIVE,
+                             MBOX::MOD_MBOXSRV_HNDLR,
+                             MBOX::RC_IPC_INVALID_DATA,         // reason Code
+                             msg_q_id,                          // IPC Data
+                             0
+                            );
+
+                        errlCommit(err,MBOX_COMP_ID);
+                    }
 
                     INTR::sendEOI(iv_msgQ,msg);
                 }
@@ -2088,8 +2112,8 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
 
             // node means Hb instance number in this context
             PIR_t my_pir (KernelIpc::ipc_data_area.pir);
-            if( (my_pir.groupId == i_node)
-                && (MBOX::HB_TEST_MSGQ != i_q_id) ) //use IPC for tests
+            if ( (my_pir.groupId == i_node)
+                && (MBOX::HB_TEST_MSGQ != i_q_id)) //use IPC for tests
             {
                 // Message is to this node - don't use IPC path
                 // MBOX sp can look up msgQ
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index 8a6a11f14..1b21cd4fe 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -334,62 +334,61 @@ class MboxSPTest : public CxxTest::TestSuite
        * @brief IPC message test
        *
        */
-// TODO: RTC:150861 re-enable test when IPC is available
-//       void testIPC(void)
-//       {
-//           TRACFCOMP(g_trac_mbox,"testIPC>");
-//           errlHndl_t err = NULL;
-//           msg_t * msg = msg_allocate();
-//           msg_t * tmsg = msg_allocate();
-//           msg_q_t msgQ = msg_q_create();
-//
-//           err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
-//           if(err)
-//           {
-//               TS_FAIL("MBOX IPC: Could not register message queue");
-//               errlCommit(err,MBOX_COMP_ID);
-//           }
-//
-//           msg->type = 1;
-//           msg->data[0] = 0x1111111111111111ull;
-//           msg->data[1] = 0x4444444444444444ull;
-//           msg->extra_data = (void *)0x3333333333333333ull;
-//
-//           *tmsg = *msg; // save for compare
-//           uint64_t node = 0;
-//
-//           err = MBOX::send(MBOX::HB_TEST_MSGQ,msg,node);
-//           if(err)
-//           {
-//               TS_FAIL("MBOX::send (IPC) returned and error log");
-//               errlCommit(err,MBOX_COMP_ID);
-//               msg_free(msg);
-//               return;
-//           }
-//
-//           msg_t * rmsg = msg_wait(msgQ);
-//
-//           if(rmsg->type != tmsg->type ||
-//              rmsg->data[0] != tmsg->data[0] ||
-//              rmsg->data[1] != tmsg->data[1] ||
-//              rmsg->extra_data != tmsg->extra_data)
-//           {
-//               TS_FAIL("IPCTEST: Unexpected IPC message");
-//
-//               TRACFCOMP(g_trac_mbox,
-//                         "IPCTEST IPC MSG: %d %lx %lx %p",
-//                         rmsg->type,
-//                         rmsg->data[0],
-//                         rmsg->data[1],
-//                         rmsg->extra_data);
-//           }
-//
-//           msgQ = MBOX::msgq_unregister(MBOX::HB_TEST_MSGQ);
-//           msg_q_destroy(msgQ);
-//           msg_free(rmsg);
-//           msg_free(tmsg);
-//           TRACFCOMP(g_trac_mbox,"<testIPC");
-//       }
+       void testIPC(void)
+       {
+           TRACFCOMP(g_trac_mbox,"testIPC>");
+           errlHndl_t err = NULL;
+           msg_t * msg = msg_allocate();
+           msg_t * tmsg = msg_allocate();
+           msg_q_t msgQ = msg_q_create();
+
+           err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
+           if(err)
+           {
+               TS_FAIL("MBOX IPC: Could not register message queue");
+               errlCommit(err,MBOX_COMP_ID);
+           }
+
+           msg->type = 1;
+           msg->data[0] = 0x1111111111111111ull;
+           msg->data[1] = 0x4444444444444444ull;
+           msg->extra_data = (void *)0x3333333333333333ull;
+
+           *tmsg = *msg; // save for compare
+           uint64_t node = 0;
+
+           err = MBOX::send(MBOX::HB_TEST_MSGQ,msg,node);
+           if(err)
+           {
+               TS_FAIL("MBOX::send (IPC) returned and error log");
+               errlCommit(err,MBOX_COMP_ID);
+               msg_free(msg);
+               return;
+           }
+
+           msg_t * rmsg = msg_wait(msgQ);
+
+           if(rmsg->type != tmsg->type ||
+              rmsg->data[0] != tmsg->data[0] ||
+              rmsg->data[1] != tmsg->data[1] ||
+              rmsg->extra_data != tmsg->extra_data)
+           {
+               TS_FAIL("IPCTEST: Unexpected IPC message");
+
+               TRACFCOMP(g_trac_mbox,
+                         "IPCTEST IPC MSG: %d %lx %lx %p",
+                         rmsg->type,
+                         rmsg->data[0],
+                         rmsg->data[1],
+                         rmsg->extra_data);
+           }
+
+           msgQ = MBOX::msgq_unregister(MBOX::HB_TEST_MSGQ);
+           msg_q_destroy(msgQ);
+           msg_free(rmsg);
+           msg_free(tmsg);
+           TRACFCOMP(g_trac_mbox,"<testIPC");
+       }
 };
 
 #endif