Multi-Drawer (IPC) Interrupt/Messaging Support

 - Use doorbells instead of IPIs (no IPI support
   using LSI interupts in the XIVE intr architecture)

 - New message type from kernel to userspace so the
   kernel can notify the HB userspace Interrupt
   Resource Provider (INTRP) that an IPC message was
   sent to the particular HB instance (in P8 this
   happened automatically as that was part of the
   IPI architecture).

 - Re-enable testcase that validates that an IPC
   message can be successfully sent.

Change-Id: Ic846f8dca45217205ed61d8381a573e995cb16f2
RTC: 150861
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/52004
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 39 insertions, 27 deletions

diff --git a/src/kernel/doorbell.C b/src/kernel/doorbell.C
index 4b99e7aee..010b7e078 100644
--- a/src/kernel/doorbell.C
+++ b/src/kernel/doorbell.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -79,17 +79,14 @@ void send_doorbell_wakeup(uint64_t i_pir)
     doorbell_send(i_pir);
 }
 
-/*
-TODO RTC 150861
 void send_doorbell_ipc(uint64_t i_pir)
 {
-    cpu_t *l_cpu = getCpu(i_pir)
+    cpu_t *l_cpu = CpuManager::getCpu(i_pir);
 
-    printk("send_doorbell_ipc to pir: %lx\n", i_pir);
+    printkd("send_doorbell_ipc to pir: %lx\n", i_pir);
     //Create WorkItem and put on the stack to be executed during doorbell
-    // execution (if needed, otherwise can likely delete and just send doorbell)
-    KernelWorkItem* l_work = new IpcDoorbellWorkItem();
+    //   execution
+    KernelWorkItem* l_work = new IPCDoorbellWorkItem();
     l_cpu->doorbell_actions.push(l_work);
-    doorbell_send(i_pir)
+    doorbell_send(i_pir);
 }
-**/
diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C
index ea055ceed..b4b5254a7 100644
--- a/src/kernel/intmsghandler.C
+++ b/src/kernel/intmsghandler.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -165,7 +165,7 @@ void InterruptMsgHdlr::addCpuCore(uint64_t i_pir)
     {
         // To avoid conflict with interrupts on thread i_pir, change the key
         // for the message to be an invalid PIR.
-        uint64_t pir_key = i_pir | 0x4000000000000000ul;
+        uint64_t pir_key = i_pir | MSG_KEY_ADD_CPU_CORE;
 
         cv_instance->iv_lock.lock();
         cv_instance->sendMessage(MSG_INTR_ADD_CPU,
@@ -180,7 +180,7 @@ void InterruptMsgHdlr::sendThreadWakeupMsg(uint64_t i_pir)
     {
         // To avoid conflict with interrupts on thread i_pir, change the key
         // for the message to be an invalid PIR.
-        uint64_t pir_key = i_pir | 0x8000000000000000ul;
+        uint64_t pir_key = i_pir | MSG_KEY_THREAD_WKUP;
 
         cv_instance->iv_lock.lock();
         cv_instance->sendMessage(MSG_INTR_CPU_WAKEUP,
@@ -189,6 +189,21 @@ void InterruptMsgHdlr::sendThreadWakeupMsg(uint64_t i_pir)
     }
 }
 
+void InterruptMsgHdlr::sendIpcMsg(uint64_t i_pir)
+{
+    if(cv_instance)
+    {
+        // To avoid conflict with interrupts on thread i_pir, change the key
+        // for the message to be an invalid PIR.
+        uint64_t pir_key = i_pir | MSG_KEY_IPC_MSG;
+
+        cv_instance->iv_lock.lock();
+        cv_instance->sendMessage(MSG_INTR_IPC,
+                                (void*)pir_key,(void *)i_pir,NULL);
+        cv_instance->iv_lock.unlock();
+    }
+}
+
 MessageHandler::HandleResult InterruptMsgHdlr::handleResponse
 (
  msg_sys_types_t i_type,
diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C
index e7a9ead9a..f7df4aacb 100644
--- a/src/kernel/ipc.C
+++ b/src/kernel/ipc.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -28,6 +28,7 @@
 #include <kernel/intmsghandler.H>
 #include <kernel/console.H>
 #include <errno.h>
+#include <kernel/doorbell.H>
 
 using namespace KernelIpc;
 
@@ -55,10 +56,10 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     // single node system with an alt-master could be designed such that the
     // alt-master were on a different logical (power bus numbering) node.
     // Since it's not in plan to use this IPC mechanism in a single node
-    // system, this case will ge ignored for now.
+    // system, this case will get ignored for now.
     uint64_t this_node = getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
     uint64_t hrmor_offset = getHRMOR()-(this_node*(ipc_data_area.hrmor_base));
-    uint64_t dest_node = (i_q >> 32) & 0x07; 
+    uint64_t dest_node = (i_q >> 32) & 0x07;
     uint64_t dest_hrmor = (ipc_data_area.hrmor_base*dest_node) + hrmor_offset;
 
     uint64_t dest_addr = reinterpret_cast<uint64_t>(&ipc_data_area);
@@ -68,7 +69,7 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     printkd("IPC Dest addr %lx Q_id:%lx\n",dest_addr,i_q);
 
     // pointer to the ipc_data_area in the destination node
-    ipc_data_area_t * p_dest = 
+    ipc_data_area_t * p_dest =
         reinterpret_cast<ipc_data_area_t*>(dest_addr);
 
     // get lock on IPC data area in other node
@@ -87,11 +88,8 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
 
     printkd("IPC send from PIR %lx to PIR %x\n",getPIR(),p_dest->pir);
 
-    /* TODO RTC 150861
-    // send IPI - use this_node + 10 as favor level of interrupt
-    //P8 Call: InterruptMsgHdlr::sendIPI(p_dest->pir,this_node + 0x10);
-    //P9 Call (likely): send_doorbell_ipc(p_dest->pir);
-    **/
+    // send doorbell to interrupt the other drawer
+    send_doorbell_ipc(p_dest->pir);
 
     // The message allocation is freed here to make msg_send for IPC
     // messages behave the same as non-IPC msg_send; that is, the message
diff --git a/src/kernel/workitem.C b/src/kernel/workitem.C
index a883cdb8f..a66908514 100644
--- a/src/kernel/workitem.C
+++ b/src/kernel/workitem.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -41,10 +41,12 @@ void CpuWakeupDoorbellWorkItem::operator() (void)
     return;
 }
 
-/*TODO RTC 150861
-void IpcDoorbellWorkItem::operator() (void)
+void IPCDoorbellWorkItem::operator() (void)
 {
-    //Decide what needs to be done when the Ipc Doorbell is received
-    // (if anything)
+    size_t pir = getPIR();
+    printk("IPC msg pir %ld done\n", pir);
+    //Send message tot he intrrp in userspace indicating it has a pending IPC
+    // message.
+    InterruptMsgHdlr::sendIpcMsg(pir);
+    return;
 }
-**/