Use non blocking messages in intRP to prevent trace deadlock

Change-Id: I14cd978a7a102f75e4829d5daa369946e766e874
CQ: SW274049
Backport: release-fips820
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/13025
Tested-by: Jenkins Server
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

6 files changed, 141 insertions, 43 deletions

diff --git a/src/include/usr/intr/interrupt.H b/src/include/usr/intr/interrupt.H
index c85cf8207..fca2ad7c3 100644
--- a/src/include/usr/intr/interrupt.H
+++ b/src/include/usr/intr/interrupt.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -147,6 +149,8 @@ namespace INTR
         MSG_INTR_MPIPL_CLEANUP,     //!< Clean up interrupts on MPIPL
         MSG_INTR_ADD_CPU_TIMEOUT,   //!< Check for a timeout waiting for a core.
         MSG_INTR_ADD_HBNODE,        //!< Add node info for MPIPL
+        MSG_INTR_EOI,               //!< Issue EOI when received
+        MSG_INTR_DRAIN_QUEUE,       //!< Allow intrp to drain Q of EOI
     };
 
     /**
@@ -167,8 +171,12 @@ namespace INTR
      * @note when an interrupt of type i_msg_type occurrs, the
      * interrupt presenter sends a sync message with type i_msg_type to
      * i_msgQ with FULL IRSN word 0 and then waits for a response.
-     * the full IRSN has the node/chip/ISN, not just the ISN
+     * Once the underlying hardware has been cleaned up (not driving
+     * the interrupt) then sendEOI should be called.  Note the msg
+     * must not be modified by the recipient.  The full IRSN has the
+     * node/chip/ISN, not just the ISN
      *       @see makeXISR
+
      *
      * @note When HB is shutting down the interrupt presenter will send
      * a message to all registered queues with a sync message type of
@@ -179,6 +187,19 @@ namespace INTR
                             ext_intr_t i_intr_type);
 
     /**
+     * Responds to the interrupt message and lets the interrupt
+     * resource provide know that it can send the End Of Interrupt
+     * to the hardware.  If the interrupt is being driven at this point
+     * a new interrupt will be fired with an additional message placed
+     * on the client's Queue
+     * @param[in] i_q message queue
+     * @param[in] i_msg The message that was sent to indicate an
+                        interrupt.  This must remain unchanged
+     *                  between receipt and sending the EOI
+     */
+    void sendEOI(msg_q_t i_q, msg_t* i_msg);
+
+    /**
      * Un register a message queue from the interrupt handler
      * @param[in] i_type the type of interrupt (ISN value)
      * @return The message queue that was unregistered with i_type
@@ -221,6 +242,13 @@ namespace INTR
      */
     errlHndl_t addHbNode(uint64_t i_hbNode);
 
+    /**
+     * Method to send synchronous message to interrupt RP.  Intent
+     * is to allow any pending EOIs (since they are async) to
+     * drain     */
+    void drainQueue(void);
+
+
 };
 
 #endif
diff --git a/src/usr/diag/attn/attnsvc.C b/src/usr/diag/attn/attnsvc.C
index 42775f555..db130ac30 100644
--- a/src/usr/diag/attn/attnsvc.C
+++ b/src/usr/diag/attn/attnsvc.C
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -389,7 +391,7 @@ void Service::processIntrQMsg(msg_t & i_msg)
     // respond to the interrupt service so hw
     // can generate additional interrupts
 
-    msg_respond(iv_intrTaskQ, &i_msg);
+    INTR::sendEOI(iv_intrTaskQ, &i_msg);
 
     // wake up the prd task
 
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 3ea39accc..9e648c98f 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -49,6 +51,7 @@
 #include <hwpf/plat/fapiPlatAttributeService.H>
 #include <mbox/mbox_queues.H>            // HB_ISTEP_MSGQ
 #include <mbox/mboxif.H>                 // register mailbox
+#include <intr/interrupt.H>
 #include <isteps/istepmasterlist.H>
 #include "istepdispatcher.H"
 #include "istep_mbox_msgs.H"
@@ -1484,6 +1487,11 @@ void IStepDispatcher::handleProcFabIovalidMsg(msg_t * & io_pMsg)
                 // keep going, since we already responded back to the FSP
             }
 
+            //All interrupt sources are blocked, but intrp could have
+            //pending EOI in message queue.  Send message to drain
+            //the interrupt queue
+            INTR::drainQueue();
+
             TRACFCOMP( g_trac_initsvc, "winkle all cores");
             uint32_t l_rc = cpu_all_winkle();
             if ( l_rc )
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index af3c3fc9b..8959b7e79 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -344,7 +346,8 @@ void IntrRp::msgHandler()
                     // Passed in as upper word of data[0]
                     uint32_t xirr = static_cast<uint32_t>(msg->data[0]>>32);
                     // data[0] (lower word) has the PIR
-                    uint64_t l_data0 = (msg->data[0] & 0xFFFFFFFF);
+                    uint64_t l_xirr_pir = msg->data[0];
+                    uint64_t l_data0 = (l_xirr_pir & 0xFFFFFFFF);
                     PIR_t pir = static_cast<PIR_t>(l_data0);
 
                     uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(pir);
@@ -373,10 +376,10 @@ void IntrRp::msgHandler()
                         msg_t * rmsg = msg_allocate();
                         rmsg->type = r->second.msgType;
                         rmsg->data[0] = type;  // interrupt type
-                        rmsg->data[1] = 0;
+                        rmsg->data[1] = l_xirr_pir;
                         rmsg->extra_data = NULL;
 
-                        int rc = msg_sendrecv(msgQ,rmsg);
+                        int rc = msg_sendrecv_noblk(msgQ,rmsg, iv_msgQ);
                         if(rc)
                         {
                             TRACFCOMP(g_trac_intr,ERR_MRK
@@ -385,7 +388,11 @@ void IntrRp::msgHandler()
                                       " handler. Ignoring it. rc = %d",
                                       (uint32_t) type, rc);
                         }
-                        msg_free(rmsg);
+
+                        //Since non IPI EOIs are blocking fashion need to open
+                        //up the interrupt priority to max (CPRR)
+                        uint8_t *cppr = reinterpret_cast<uint8_t*>(xirrAddress);
+                        *cppr = CPPR_ENABLE_ALL; //allow any INTR
                     }
                     else if (type == INTERPROC_XISR)
                     {
@@ -447,35 +454,34 @@ void IntrRp::msgHandler()
                             }
 
                         }
-                    }
 
-                    // Writing the XIRR with the same value read earlier
-                    // tells the interrupt presenter hardware to signal an EOI.
-                    xirr |= CPPR_MASK;  //set all CPPR bits - allow any INTR
-                    *xirrAddress = xirr;
+                        // Writing the XIRR with the same value read earlier
+                        // to signal an EOI.
+                        xirr |= CPPR_MASK;  //set all CPPR bits - allow any INTR
+                        *xirrAddress = xirr;
 
-                    TRACDCOMP(g_trac_intr,
-                              "EOI issued. XIRR=%x, PIR=%x",
-                              xirr,pir);
+                        TRACDCOMP(g_trac_intr,
+                                  "EOI issued. XIRR=%x, PIR=%x",
+                                  xirr,pir);
 
-                    // Now handle any IPC messages
-                    if (type == INTERPROC_XISR)
-                    {
-                        // If something is registered for IPIs and
-                        // it has not already been handled then handle
+                        // Now handle any IPC messages
+                        // If something is registered for IPIs
+                        // and msg is ready, then handle
                         if(r != iv_registry.end() &&
-                           KernelIpc::ipc_data_area.msg_queue_id !=
-                           IPC_DATA_AREA_READ)
+                           (KernelIpc::ipc_data_area.msg_queue_id !=
+                           IPC_DATA_AREA_CLEAR) &&
+                            (KernelIpc::ipc_data_area.msg_queue_id !=
+                             IPC_DATA_AREA_LOCKED))
                         {
                             msg_q_t msgQ = r->second.msgQ;
 
                             msg_t * rmsg = msg_allocate();
                             rmsg->type = r->second.msgType;
                             rmsg->data[0] = type;  // interrupt type
-                            rmsg->data[1] = 0;
+                            rmsg->data[1] = l_xirr_pir;
                             rmsg->extra_data = NULL;
 
-                            int rc = msg_sendrecv(msgQ,rmsg);
+                            int rc = msg_sendrecv_noblk(msgQ, rmsg, iv_msgQ);
                             if(rc)
                             {
                                 TRACFCOMP(g_trac_intr,ERR_MRK
@@ -484,19 +490,41 @@ void IntrRp::msgHandler()
                                           "handler. Ignoring it. rc = %d",
                                           rc);
                             }
-                            msg_free(rmsg);
-                        }
-                        if(KernelIpc::ipc_data_area.msg_queue_id ==
-                           IPC_DATA_AREA_READ)
-                        {
-                            KernelIpc::ipc_data_area.msg_queue_id =
-                                IPC_DATA_AREA_CLEAR;
                         }
                     }
+                }
+                break;
 
+            case MSG_INTR_EOI:
+                {
+                    // Write the XIRR with the same value read earlier
+                    // to signal EOI. XIRR value was stored in data[1]
+                    // Only do this for non IPI types (type is in data[0])
+                    if(msg->data[0] != INTERPROC_XISR)
+                    {
+                        // Passed in as upper word of data[1]
+                        uint32_t xirr = static_cast<uint32_t>(msg->data[1]>>32);
+                        // data[1] (lower word) has the PIR
+                        uint64_t l_data0 = (msg->data[1] & 0xFFFFFFFF);
+                        PIR_t pir = static_cast<PIR_t>(l_data0);
+
+                        uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(pir);
+                        uint32_t * xirrAddress =
+                          reinterpret_cast<uint32_t*>(baseAddr + XIRR_OFFSET);
+
+                        xirr |= CPPR_MASK;  //set all CPPR bits - allow any INTR
+                        *xirrAddress = xirr;
+
+                        TRACDCOMP(g_trac_intr,
+                                  "EOI issued. XIRR=%x, PIR=%x",
+                                  xirr,pir);
+                    }
+                    msg_free(msg);
                 }
                 break;
 
+
+
             case MSG_INTR_REGISTER_MSGQ:
                 {
                     msg_q_t l_msgQ = reinterpret_cast<msg_q_t>(msg->data[0]);
@@ -706,6 +734,15 @@ void IntrRp::msgHandler()
                 }
                 break;
 
+            case MSG_INTR_DRAIN_QUEUE:
+                {
+                    //The purpose of this message is allow the
+                    //intrp to drain its message queue of pending EOIs
+                    //just respond
+                    msg_respond(iv_msgQ,msg);
+                }
+                break;
+
             default:
                 msg->data[1] = -EINVAL;
                 msg_respond(iv_msgQ, msg);
@@ -2502,6 +2539,14 @@ errlHndl_t INTR::registerMsgQ(msg_q_t i_msgQ,
     return err;
 }
 
+void INTR::sendEOI(msg_q_t i_q, msg_t* i_msg)
+{
+    //Fix up message to make it easier to handle
+    //Users are required to NOT touch it
+    i_msg->type = MSG_INTR_EOI;
+    msg_respond(i_q,i_msg);
+}
+
 // Unregister message queue from interrupt handler
 msg_q_t INTR::unRegisterMsgQ(ext_intr_t i_type)
 {
@@ -2741,3 +2786,21 @@ errlHndl_t INTR::addHbNode(uint64_t i_hbNode)
     return err;
 }
 
+/*
+ * Drain interrupt message queue (if present)
+ */
+void INTR::drainQueue()
+{
+    // send a sync message if queue is found
+    msg_q_t intr_msgQ = msg_q_resolve(VFS_ROOT_MSG_INTR);
+    if(intr_msgQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->type = MSG_INTR_DRAIN_QUEUE;
+
+        msg_sendrecv(intr_msgQ, msg);
+
+        msg_free(msg);
+    }
+    //else no queue, no need to do anything
+}
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 314ae5217..2bbb6d8a7 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -116,6 +116,7 @@ namespace INTR
                 LINKC_OFFSET = 24,             //!< offset to LINKC register
                 XISR_MASK    = 0x00FFFFFF,     //!< XISR MASK in XIRR register
                 CPPR_MASK    = 0xFF000000,     //!< CPPR MASK in XIRR register
+                CPPR_ENABLE_ALL = 0xFF,        //!< All interrupt priorities
 
                 ICPBAR_EN    = 30,              //!< BAR enable bit pos
                 ICPBAR_SCOM_ADDR = 0x020109ca,  //!< ICP BAR scom address
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index e8a9c8865..abcc93b76 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -239,9 +239,7 @@ void MailboxSp::msgHandler()
                     err = handleInterrupt();
 
                     // Respond to the interrupt handler regardless of err
-                    msg->data[0] = 0;
-                    msg->data[1] = 0;
-                    msg_respond(iv_msgQ,msg);
+                    INTR::sendEOI(iv_msgQ,msg);
 
                     // err will be set if scom failed in mbox DD
                     // or MBOX_DATA_WRITE_ERR  - serious - assert
@@ -408,16 +406,14 @@ void MailboxSp::msgHandler()
                         isync();
                         *ipc_msg = KernelIpc::ipc_data_area.msg_payload;
                         lwsync();
-                        // Signal message has been read, but keep area locked
-                        // until eoi has been sent
+                        // EOI has already been sent by intrp. Simply clear
+                        // ipc area so another message can be sent
                         KernelIpc::ipc_data_area.msg_queue_id =
-                            IPC_DATA_AREA_READ;
+                            IPC_DATA_AREA_CLEAR;
                         handleIPC(static_cast<queue_id_t>(msg_q_id), ipc_msg);
                     }
 
-                    msg->data[0] = 0;
-                    msg->data[1] = 0;
-                    msg_respond(iv_msgQ,msg);
+                    INTR::sendEOI(iv_msgQ,msg);
                 }
 
                 break;