Missing HB interrupts

CQ: SW244546
Change-Id: Ice49cdd7e4acd8168ec2cfc29e1970c5daa49780
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/8666
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

8 files changed, 70 insertions, 26 deletions

diff --git a/src/include/kernel/intmsghandler.H b/src/include/kernel/intmsghandler.H
index 220a7fa30..02f681bd0 100644
--- a/src/include/kernel/intmsghandler.H
+++ b/src/include/kernel/intmsghandler.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -143,8 +143,11 @@ class InterruptMsgHdlr : public MessageHandler
          * Issue an IPI to the core.
          *
          * @param[in] i_pir - The PIR of the CPU to send IPI at.
+         * @param[in] i_favor - How favored the interrupt is; 0 = most favored
+         *                      254 = least favored, 255 = no interrupt
+         *                      Default 0x1 - IPI from kernel side (wakeup)
          */
-        static void sendIPI(uint64_t i_pir);
+        static void sendIPI(uint64_t i_pir, uint8_t i_favor = 0x1);
 
     private:
 
diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C
index d3e6a8298..1443847e1 100644
--- a/src/kernel/intmsghandler.C
+++ b/src/kernel/intmsghandler.C
@@ -173,7 +173,7 @@ void InterruptMsgHdlr::addCpuCore(uint64_t i_pir)
     }
 }
 
-void InterruptMsgHdlr::sendIPI(uint64_t i_pir)
+void InterruptMsgHdlr::sendIPI(uint64_t i_pir, uint8_t i_favor)
 {
     uint64_t mfrrAddress = cv_ipc_base_address;
     mfrrAddress += mmio_offset(i_pir);
@@ -181,7 +181,7 @@ void InterruptMsgHdlr::sendIPI(uint64_t i_pir)
 
     mfrrAddress |= 0x8000000000000000ul;
 
-    register uint8_t data = 0;
+    register uint8_t data = i_favor;
 
     eieio(); sync();
     MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE);
diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C
index 6a63225fc..4741a381c 100644
--- a/src/kernel/ipc.C
+++ b/src/kernel/ipc.C
@@ -85,8 +85,8 @@ void KernelIpc::send(uint64_t i_q, msg_t * i_msg)
 
     printkd("IPC send from PIR %lx to PIR %x\n",getPIR(),p_dest->pir);
 
-    // send IPI
-    InterruptMsgHdlr::sendIPI(p_dest->pir);
+    // send IPI - use this_node + 10 as favor level of interrupt
+    InterruptMsgHdlr::sendIPI(p_dest->pir,this_node + 10);
 
     // The message allocation is freed here to make msg_send for IPC
     // messages behave the same as non-IPC msg_send; that is, the message
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index 815c029d1..d109b31fc 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -37,6 +37,7 @@
 #include <kernel/intmsghandler.H>
 #include <kernel/hbdescriptor.H>
 #include <kernel/ipc.H>
+#include <kernel/timemgr.H>
 
 extern "C"
     void kernel_shutdown(size_t, uint64_t, uint64_t, uint64_t,
@@ -224,7 +225,7 @@ namespace KernelMisc
 
         // Save away the current timebase.  All threads are in this object
         // now so they're not going to be using the time for anything else.
-        iv_timebase = getTB();
+        iv_timebase = getTB() + TimeManager::convertSecToTicks(1,0);
     }
 
     extern "C" void kernel_execute_winkle(task_t* t);
@@ -309,7 +310,7 @@ namespace KernelMisc
 
         // Save away the current timebase.  All threads are in this object
         // now so they're not going to be using the time for anything else.
-        iv_timebase = getTB();
+        iv_timebase = getTB() + TimeManager::convertSecToTicks(1,0);
     }
 
     void WinkleAll::activeMainWork()
diff --git a/src/usr/hwpf/hwp/establish_system_smp/establish_system_smp.C b/src/usr/hwpf/hwp/establish_system_smp/establish_system_smp.C
index f33475873..228e82383 100644
--- a/src/usr/hwpf/hwp/establish_system_smp/establish_system_smp.C
+++ b/src/usr/hwpf/hwp/establish_system_smp/establish_system_smp.C
@@ -86,7 +86,7 @@ using   namespace   EDI_EI_INITIALIZATION;
 /******************************************************************************/
 const uint8_t HB_COALESCE_WAITING_FOR_MSG = 0x0;
 const uint8_t HB_COALESCE_MSG_DONE = 0x1;
-const uint8_t MAX_TIME_ALLOWED_MS = 10;
+const uint32_t MAX_TIME_ALLOWED_MS = 10000;
 const uint8_t NUMBER_OF_POSSIBLE_NODES = 8;
 const uint8_t CONTINUE_WAIT_FOR_MSGS = 0x2;
 const uint8_t TIME_EXPIRED=0x3;
@@ -101,7 +101,7 @@ void* host_coalese_timer(void* i_msgQPtr)
 
     msg_t* msg = msg_allocate();
     msg->type = HOST_COALESCE_TIMER_MSG;
-    uint8_t l_time_ms =0;
+    uint32_t l_time_ms =0;
 
     msg_q_t* msgQ = static_cast<msg_q_t*>(i_msgQPtr);
 
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index d1f114153..c9b3f7edb 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -362,15 +362,16 @@ void IntrRp::msgHandler()
                     type = static_cast<ext_intr_t>(xirr & XISR_MASK);
 
                     TRACFCOMP(g_trac_intr,
-                              "External Interrupt recieved, Type=%x",
-                              type);
+                              "External Interrupt recieved. XIRR=%x, PIR=%x",
+                              xirr,pir);
 
                     // Acknowlege msg
                     msg->data[1] = 0;
                     msg_respond(iv_msgQ, msg);
 
                     Registry_t::iterator r = iv_registry.find(type);
-                    if(r != iv_registry.end())
+                    if(r != iv_registry.end() &&
+                       type != INTERPROC_XISR) //handle IPI after EOI, not here
                     {
                         msg_q_t msgQ = r->second.msgQ;
 
@@ -417,6 +418,9 @@ void IntrRp::msgHandler()
                         // Clear IPI request.
                         volatile uint8_t * mfrr =
                             reinterpret_cast<uint8_t*>(baseAddr + MFRR_OFFSET);
+
+                        TRACFCOMP( g_trac_intr,"mfrr = %x",*mfrr);
+
                         (*mfrr) = 0xff;
                         eieio();  // Force mfrr clear before xirr EIO.
 
@@ -452,11 +456,37 @@ void IntrRp::msgHandler()
 
                     // Writing the XIRR with the same value read earlier
                     // tells the interrupt presenter hardware to signal an EOI.
+                    xirr |= CPPR_MASK;  //set all CPPR bits - allow any INTR
                     *xirrAddress = xirr;
 
-                    // indicate IPC data area clear after EOI has been sent
+                    // Now handle any IPC messages
                     if (type == INTERPROC_XISR)
                     {
+                        // If something is registered for IPIs and
+                        // it has not already been handled then handle
+                        if(r != iv_registry.end() && 
+                           KernelIpc::ipc_data_area.msg_queue_id !=
+                           IPC_DATA_AREA_READ)
+                        {
+                            msg_q_t msgQ = r->second.msgQ;
+
+                            msg_t * rmsg = msg_allocate();
+                            rmsg->type = r->second.msgType;
+                            rmsg->data[0] = type;  // interrupt type
+                            rmsg->data[1] = 0;
+                            rmsg->extra_data = NULL;
+
+                            int rc = msg_sendrecv(msgQ,rmsg);
+                            if(rc)
+                            {
+                                TRACFCOMP(g_trac_intr,ERR_MRK
+                                          "IPI Interrupt recieved, but could "
+                                          "not send message to the registered "
+                                          "handler. Ignoring it. rc = %d",
+                                          rc);
+                            }
+                            msg_free(rmsg);
+                        }
                         if(KernelIpc::ipc_data_area.msg_queue_id ==
                            IPC_DATA_AREA_READ)
                         {
@@ -1205,16 +1235,8 @@ void IntrRp::initInterruptPresenter(const PIR_t i_pir) const
     TRACDCOMP(g_trac_intr,"PIR 0x%x offset: 0x%lx",
               i_pir.word,
               cpuOffsetAddr(i_pir));
-    if(i_pir.word == iv_masterCpu.word)
-    {
-        *cppr = 0xff;      // Allow all interrupts on master.
-    }
-    else
-    {
-        *cppr = 0x01;      // Allow only priority 0 interrupts on non-masters.
-                           // We use priority 0 to deliver IPIs for waking up
-                           // a core.
-    }
+
+    *cppr = 0xff;          // Allow all interrupts
 
     // Links are intended to be set up in rings.  If an interrupt ends up
     // where it started, it gets rejected by hardware.
@@ -1285,7 +1307,7 @@ void IntrRp::sendIPI(const PIR_t i_pir) const
 
     eieio(); sync();
     MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE);
-    (*mfrr) = 0x00;
+    (*mfrr) = IPI_USR_PRIO;
 }
 
 
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 17dcee41e..650479b8f 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -112,6 +112,7 @@ namespace INTR
                 LINKB_OFFSET = 20,             //!< offset to LINKB register
                 LINKC_OFFSET = 24,             //!< offset to LINKC register
                 XISR_MASK    = 0x00FFFFFF,     //!< XISR MASK in XIRR register
+                CPPR_MASK    = 0xFF000000,     //!< CPPR MASK in XIRR register
 
                 ICPBAR_EN    = 30,              //!< BAR enable bit pos
                 ICPBAR_SCOM_ADDR = 0x020109ca,  //!< ICP BAR scom address
@@ -153,6 +154,8 @@ namespace INTR
 
                 PSI_FSP_INT_ENABLE      = 0x1000000000000000ULL,
                 PSI_HBCR_AND_SCOM_ADDR  = 0x02010913,
+
+                IPI_USR_PRIO        = 0x2,   //<! IPI priority from USR space
             };
 
             enum INTR_ROUTING_t
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index de3a8e23e..db56c6678 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -1651,8 +1651,23 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
         {
             uint64_t q_handle = i_q_id;
             q_handle |= (((uint64_t)MSGQ_TYPE_IPC | (uint64_t)i_node) << 32);
+            TRACFCOMP(g_trac_mboxmsg,INFO_MRK
+                      "MBOXSP IPC SEND MSG: Dest node %d. msg_id: %lx",
+                      i_node,
+                      (uint32_t)i_q_id);
+
+            TRACFCOMP(g_trac_mboxmsg,INFO_MRK
+                      "MBOXSP IPC SEND MSG: 0x%08x 0x%016lx 0x%016lx %p",
+                      i_msg->type,
+                      i_msg->data[0],
+                      i_msg->data[1],
+                      i_msg->extra_data);
+
             int rc = msg_send(reinterpret_cast<msg_q_t>(q_handle),
                               i_msg);
+            TRACFCOMP(g_trac_mboxmsg,INFO_MRK"MBOXSP IPC SENT. This PIR 0x%x",
+                      KernelIpc::ipc_data_area.pir);
+
 
             if(rc)
             {