Multi-Drawer (IPC) Interrupt/Messaging Support

 - Use doorbells instead of IPIs (no IPI support
   using LSI interupts in the XIVE intr architecture)

 - New message type from kernel to userspace so the
   kernel can notify the HB userspace Interrupt
   Resource Provider (INTRP) that an IPC message was
   sent to the particular HB instance (in P8 this
   happened automatically as that was part of the
   IPI architecture).

 - Re-enable testcase that validates that an IPC
   message can be successfully sent.

Change-Id: Ic846f8dca45217205ed61d8381a573e995cb16f2
RTC: 150861
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/52004
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

5 files changed, 148 insertions, 123 deletions

diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 69abb9102..a164544d8 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] Google Inc.                                                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index e961f6395..a076e0fb5 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -930,14 +930,18 @@ void IntrRp::msgHandler()
                     }
                 }
                 break;
-/*TODO RTC 150861 -- I think a new IPC message type needs to be defined.
- *                   And the code below should be executed when this new message
- *                   type is received. The Kernel will send this message to
- *                   here (this intrrp code) during doorbell wakeup.
+            case MSG_INTR_IPC:
+                {
+                    errlHndl_t l_err = NULL;
+                    uint64_t l_xirr_pir = msg->data[0];
+
+                    TRACFCOMP(g_trac_intr,INFO_MRK
+                              "IntrRp::msgHandler Doorbell IPC msg received"
+                                  " for %d", l_xirr_pir);
 
                     // Now handle any IPC messages
-                    // If something is registered for IPIs
-                    // and msg is ready, then handle
+                    // If something is registered for the IPC msg
+                    Registry_t::iterator r = iv_registry.find(ISN_INTERPROC);
                     if(r != iv_registry.end() &&
                        (KernelIpc::ipc_data_area.msg_queue_id !=
                        IPC_DATA_AREA_CLEAR) &&
@@ -948,7 +952,7 @@ void IntrRp::msgHandler()
 
                         msg_t * rmsg = msg_allocate();
                         rmsg->type = r->second.msgType;
-                        rmsg->data[0] = type;  // interrupt type
+                        rmsg->data[0] = ISN_INTERPROC;
                         rmsg->data[1] = l_xirr_pir;
                         rmsg->extra_data = NULL;
 
@@ -956,17 +960,57 @@ void IntrRp::msgHandler()
                         if(rc)
                         {
                             TRACFCOMP(g_trac_intr,ERR_MRK
-                                      "IPI Interrupt received, but could "
+                                      "IPC message received, but could "
                                       "not send message to the registered "
                                       "handler. Ignoring it. rc = %d",
                                       rc);
                         }
                     }
-**/
+                    else if(KernelIpc::ipc_data_area.msg_queue_id ==
+                              IPC_DATA_AREA_CLEAR ||
+                            KernelIpc::ipc_data_area.msg_queue_id ==
+                              IPC_DATA_AREA_LOCKED)
+                    {
+                        TRACFCOMP(g_trac_intr,ERR_MRK
+                                   "IPC message received but data area is in"
+                                   " an invalid state. msg_queue_id = 0x%lx",
+                                   KernelIpc::ipc_data_area.msg_queue_id);
+                        /*@ errorlog tag
+                         * @errortype       ERRL_SEV_PREDICTIVE
+                         * @moduleid        INTR::MOD_INTRRP_IPC
+                         * @reasoncode      INTR::RC_IPC_DATA_INVALID
+                         * @userdata1       IPC Data Area MSG Queue ID
+                         * @userdata2       PIR
+                         * @devdesc         Error encountered routing IPC
+                         *                  message
+                         */
+                        l_err = new ERRORLOG::ErrlEntry
+                            (
+                             ERRORLOG::ERRL_SEV_PREDICTIVE,    // severity
+                             INTR::MOD_INTRRP_IPC,             // moduleid
+                             INTR::RC_IPC_DATA_INVALID,        // reason code
+                             KernelIpc::ipc_data_area.msg_queue_id,
+                             l_xirr_pir
+                            );
+                    }
+                    else
+                    {
+                        TRACFCOMP(g_trac_intr,ERR_MRK
+                           "IPC Message received type but nothing registered "
+                           "to handle it. Ignoring it.");
+                    }
+
+                    if (l_err)
+                    {
+                        l_err->collectTrace(INTR_COMP_NAME, 256);
+                        errlCommit(l_err, INTR_COMP_ID);
+                    }
+                }
+                break;
             case MSG_INTR_EOI:
                 {
                     // Use standrard EOI (End of Interrupt) sequence
-                    if(msg->data[0] != INTERPROC_XISR)
+                    if(msg->data[0] != ISN_INTERPROC)
                     {
                         uint64_t intSource = msg->data[0];
                         PIR_t l_pir = msg->data[1];
@@ -1342,7 +1386,7 @@ void IntrRp::routeInterrupt(intr_hdlr_t* i_proc,
     // interrupt source
     Registry_t::iterator r = iv_registry.find(i_type);
 
-    if(r != iv_registry.end() && i_type != INTERPROC_XISR)
+    if(r != iv_registry.end() && i_type != ISN_INTERPROC)
     {
         msg_q_t msgQ = r->second.msgQ;
 
@@ -1928,47 +1972,6 @@ msg_q_t IntrRp::unregisterInterruptXISR(ext_intr_t i_xisr)
     return msgQ;
 }
 
-void IntrRp::sendIPI(const PIR_t i_pir) const
-{
-    uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(i_pir);
-    volatile uint8_t * mfrr =
-        reinterpret_cast<uint8_t*>(baseAddr + MFRR_OFFSET);
-
-    eieio(); sync();
-    MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE);
-    (*mfrr) = IPI_USR_PRIO;
-}
-
-
-errlHndl_t IntrRp::checkAddress(uint64_t i_addr)
-{
-    errlHndl_t err = NULL;
-
-    if(i_addr < VMM_VADDR_DEVICE_SEGMENT_FIRST)
-    {
-        /*@ errorlog tag
-         * @errortype       ERRL_SEV_INFORMATIONAL
-         * @moduleid        INTR::MOD_INTRRP_CHECKADDRESS
-         * @reasoncode      INTR::RC_BAD_VIRTUAL_IO_ADDRESS
-         * @userdata1       The bad virtual address
-         * @userdata2       0
-         *
-         * @devdesc         The virtual address is not a valid IO address
-         *
-         */
-        err = new ERRORLOG::ErrlEntry
-            (
-             ERRORLOG::ERRL_SEV_INFORMATIONAL,
-             INTR::MOD_INTRRP_CHECKADDRESS,
-             INTR::RC_BAD_VIRTUAL_IO_ADDRESS,
-             i_addr,
-             0
-            );
-    }
-
-    return err;
-}
-
 void IntrRp::shutDown(uint64_t i_status)
 {
     msg_t * rmsg = msg_allocate();
@@ -2671,7 +2674,7 @@ void IntrRp::drainMpIplInterrupts(TARGETING::TargetHandleList & i_cores)
                     volatile uint32_t xirr_rw = *xirrPtr;
 
                     //If IPI need to set mfrr to 0xFF
-                    if(INTERPROC_XISR == xirr)
+                    if(ISN_INTERPROC == xirr)
                     {
                         *mfrrPtr = 0xFF;
                     }
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index f2f3cd26e..ce32fb233 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -216,7 +216,6 @@ void IpcSp::msgHandler()
 
                 break;
 
-
             default:
 
                 TRACFCOMP( g_trac_ipc,
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 7260d129b..7d06c2e89 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] Google Inc.                                                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
@@ -189,13 +189,10 @@ errlHndl_t MailboxSp::_init()
         }
     }
 
-    // @todo RTC:126643
-#if (0)
     // Register for IPC messages
     err = INTR::registerMsgQ(iv_msgQ,
                              MSG_IPC,
                              INTR::ISN_INTERPROC);
-#endif
 
     if(mbxComm)
     {
@@ -268,7 +265,6 @@ void MailboxSp::msgHandler()
                     printkd("MSG_INTR type - handling interrupt\n");
                     err = handleInterrupt();
 
-
                     printkd("MSG_INTR type - sending EOI\n");
                     // Respond to the interrupt handler regardless of err
                     INTR::sendEOI(iv_msgQ,msg);
@@ -447,10 +443,11 @@ void MailboxSp::msgHandler()
 
             case MSG_IPC: // Look for Interprocessor Messages
                 {
-
                     uint64_t msg_q_id = KernelIpc::ipc_data_area.msg_queue_id;
                     if(msg_q_id == IPC_DATA_AREA_LOCKED)
                     {
+                        TRACFCOMP(g_trac_mbox, INFO_MRK
+                             "MBOXSP IPC data area locked");
                         // msg is being written, but not yet ready to handle
                         msg_q_id = IPC_DATA_AREA_CLEAR;
                     }
@@ -464,12 +461,39 @@ void MailboxSp::msgHandler()
                         isync();
                         *ipc_msg = KernelIpc::ipc_data_area.msg_payload;
                         lwsync();
-                        // EOI has already been sent by intrp. Simply clear
-                        // ipc area so another message can be sent
+                        // Clear ipc area so another message can be sent
                         KernelIpc::ipc_data_area.msg_queue_id =
                             IPC_DATA_AREA_CLEAR;
                         handleIPC(static_cast<queue_id_t>(msg_q_id), ipc_msg);
                     }
+                    else
+                    {
+                        TRACFCOMP(g_trac_mbox, ERR_MRK
+                                "MBOXSP invalid data found in IPC data area: "
+                                " %0xlx", msg_q_id);
+
+                        TRACFBIN(g_trac_mbox, "IPC Data Area:",
+                              &KernelIpc::ipc_data_area,
+                              sizeof(KernelIpc::ipc_data_area));
+
+                        /*@ errorlog tag
+                         * @errortype       ERRL_SEV_PREDICTIVE
+                         * @moduleid        MBOX::MOD_MBOXSRV_HNDLR
+                         * @reasoncode      MBOX::RC_IPC_INVALID_DATA
+                         * @userdata1       IPC Data Area MSG Queue ID
+                         * @devdesc         IPC Message data corrupted
+                         */
+                        err = new ERRORLOG::ErrlEntry
+                            (
+                             ERRORLOG::ERRL_SEV_PREDICTIVE,
+                             MBOX::MOD_MBOXSRV_HNDLR,
+                             MBOX::RC_IPC_INVALID_DATA,         // reason Code
+                             msg_q_id,                          // IPC Data
+                             0
+                            );
+
+                        errlCommit(err,MBOX_COMP_ID);
+                    }
 
                     INTR::sendEOI(iv_msgQ,msg);
                 }
@@ -2088,8 +2112,8 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
 
             // node means Hb instance number in this context
             PIR_t my_pir (KernelIpc::ipc_data_area.pir);
-            if( (my_pir.groupId == i_node)
-                && (MBOX::HB_TEST_MSGQ != i_q_id) ) //use IPC for tests
+            if ( (my_pir.groupId == i_node)
+                && (MBOX::HB_TEST_MSGQ != i_q_id)) //use IPC for tests
             {
                 // Message is to this node - don't use IPC path
                 // MBOX sp can look up msgQ
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index 8a6a11f14..1b21cd4fe 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -334,62 +334,61 @@ class MboxSPTest : public CxxTest::TestSuite
        * @brief IPC message test
        *
        */
-// TODO: RTC:150861 re-enable test when IPC is available
-//       void testIPC(void)
-//       {
-//           TRACFCOMP(g_trac_mbox,"testIPC>");
-//           errlHndl_t err = NULL;
-//           msg_t * msg = msg_allocate();
-//           msg_t * tmsg = msg_allocate();
-//           msg_q_t msgQ = msg_q_create();
-//
-//           err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
-//           if(err)
-//           {
-//               TS_FAIL("MBOX IPC: Could not register message queue");
-//               errlCommit(err,MBOX_COMP_ID);
-//           }
-//
-//           msg->type = 1;
-//           msg->data[0] = 0x1111111111111111ull;
-//           msg->data[1] = 0x4444444444444444ull;
-//           msg->extra_data = (void *)0x3333333333333333ull;
-//
-//           *tmsg = *msg; // save for compare
-//           uint64_t node = 0;
-//
-//           err = MBOX::send(MBOX::HB_TEST_MSGQ,msg,node);
-//           if(err)
-//           {
-//               TS_FAIL("MBOX::send (IPC) returned and error log");
-//               errlCommit(err,MBOX_COMP_ID);
-//               msg_free(msg);
-//               return;
-//           }
-//
-//           msg_t * rmsg = msg_wait(msgQ);
-//
-//           if(rmsg->type != tmsg->type ||
-//              rmsg->data[0] != tmsg->data[0] ||
-//              rmsg->data[1] != tmsg->data[1] ||
-//              rmsg->extra_data != tmsg->extra_data)
-//           {
-//               TS_FAIL("IPCTEST: Unexpected IPC message");
-//
-//               TRACFCOMP(g_trac_mbox,
-//                         "IPCTEST IPC MSG: %d %lx %lx %p",
-//                         rmsg->type,
-//                         rmsg->data[0],
-//                         rmsg->data[1],
-//                         rmsg->extra_data);
-//           }
-//
-//           msgQ = MBOX::msgq_unregister(MBOX::HB_TEST_MSGQ);
-//           msg_q_destroy(msgQ);
-//           msg_free(rmsg);
-//           msg_free(tmsg);
-//           TRACFCOMP(g_trac_mbox,"<testIPC");
-//       }
+       void testIPC(void)
+       {
+           TRACFCOMP(g_trac_mbox,"testIPC>");
+           errlHndl_t err = NULL;
+           msg_t * msg = msg_allocate();
+           msg_t * tmsg = msg_allocate();
+           msg_q_t msgQ = msg_q_create();
+
+           err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
+           if(err)
+           {
+               TS_FAIL("MBOX IPC: Could not register message queue");
+               errlCommit(err,MBOX_COMP_ID);
+           }
+
+           msg->type = 1;
+           msg->data[0] = 0x1111111111111111ull;
+           msg->data[1] = 0x4444444444444444ull;
+           msg->extra_data = (void *)0x3333333333333333ull;
+
+           *tmsg = *msg; // save for compare
+           uint64_t node = 0;
+
+           err = MBOX::send(MBOX::HB_TEST_MSGQ,msg,node);
+           if(err)
+           {
+               TS_FAIL("MBOX::send (IPC) returned and error log");
+               errlCommit(err,MBOX_COMP_ID);
+               msg_free(msg);
+               return;
+           }
+
+           msg_t * rmsg = msg_wait(msgQ);
+
+           if(rmsg->type != tmsg->type ||
+              rmsg->data[0] != tmsg->data[0] ||
+              rmsg->data[1] != tmsg->data[1] ||
+              rmsg->extra_data != tmsg->extra_data)
+           {
+               TS_FAIL("IPCTEST: Unexpected IPC message");
+
+               TRACFCOMP(g_trac_mbox,
+                         "IPCTEST IPC MSG: %d %lx %lx %p",
+                         rmsg->type,
+                         rmsg->data[0],
+                         rmsg->data[1],
+                         rmsg->extra_data);
+           }
+
+           msgQ = MBOX::msgq_unregister(MBOX::HB_TEST_MSGQ);
+           msg_q_destroy(msgQ);
+           msg_free(rmsg);
+           msg_free(tmsg);
+           TRACFCOMP(g_trac_mbox,"<testIPC");
+       }
 };
 
 #endif