HBMBOX Retrieve all DMA pages from the FSP prior to controlled shutdown

Change-Id: I1fb2d1989679742d2497130d61afae059fa126fb
RTC: 97366
CQ: SW254551
Backport: release-fips810
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/9406
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

4 files changed, 87 insertions, 40 deletions

diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index f36d167b3..9ff2d6421 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -241,13 +241,31 @@ void MailboxSp::msgHandler()
                     {
                         errlCommit(err,MBOX_COMP_ID);
 
-                        TRACFCOMP(g_trac_mbox, ERR_MRK"MBOXSP HALTED on critical error!");
+                        TRACFCOMP(g_trac_mbox, ERR_MRK
+                                  "MBOXSP HALTED on critical error!");
                         crit_assert(0);
                     }
 
                     if(iv_shutdown_msg && quiesced())
                     {
-                        handleShutdown();
+                        // If all DMA buffers still not owned
+                        // try once to get them all back.
+                        if(!iv_dmaBuffer.ownsAllBlocks() &&
+                           !iv_dmaBuffer.shutdownDmaRequestSent())
+                        {
+                            iv_dmaBuffer.setShutdownDmaRequestSent(true);
+                            mbox_msg_t  dma_request_msg;
+                            dma_request_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
+                            dma_request_msg.msg_payload.type =
+                                MSG_REQUEST_DMA_BUFFERS;
+                            dma_request_msg.msg_payload.__reserved__async = 1;
+
+                            send_msg(&dma_request_msg);
+                        }
+                        else
+                        {
+                            handleShutdown();
+                        }
                     }
 
                     if(iv_suspended && quiesced())
@@ -295,8 +313,33 @@ void MailboxSp::msgHandler()
                         resume();
                     }
 
+                    // Deal with messages never claimed by any HB component
                     handleUnclaimed();
-                    if(quiesced())
+
+                    // State:  MBOX is quiesced() and owns all the DMA buffers
+                    // Action: handleShutdown() now
+                    //
+                    // State:  MBOX is quiesced(), but does not own all the DMA
+                    //         buffers
+                    // Action: Must send message to retrieve the buffers.
+                    //
+                    // State:  MBOX is still busy
+                    // Action: Send message to retrieve the DMA buffers if one
+                    //         is not pending, but may not get them all back
+                    //         at this time.
+                    //
+                    if(!iv_dmaBuffer.ownsAllBlocks() && !iv_dma_pend)
+                    {
+                        mbox_msg_t  dma_request_msg;
+                        dma_request_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
+                        dma_request_msg.msg_payload.type =
+                            MSG_REQUEST_DMA_BUFFERS;
+                        dma_request_msg.msg_payload.__reserved__async = 1;
+
+                        send_msg(&dma_request_msg);
+                    }
+
+                    if(quiesced()) //already in shutdown state
                     {
                         handleShutdown();       // done - shutdown now.
                     }
@@ -587,7 +630,6 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
                 iv_msg_to_send.msg_payload.data[1] = 0;
                 iv_msg_to_send.msg_payload.extra_data = NULL;
                 iv_msg_to_send.msg_payload.__reserved__async = 1;
-                iv_dma_pend = true;
             }
             else
             {
@@ -642,6 +684,12 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
         size_t mbox_msg_len = sizeof(mbox_msg_t);
         iv_rts = false;
 
+        if(iv_msg_to_send.msg_queue_id == FSP_MAILBOX_MSGQ &&
+           iv_msg_to_send.msg_payload.type == MSG_REQUEST_DMA_BUFFERS)
+        {
+            iv_dma_pend = true;
+        }
+
         trace_msg("SEND",iv_msg_to_send);
 
         err = DeviceFW::deviceWrite(iv_trgt,
@@ -1039,39 +1087,13 @@ void MailboxSp::handle_hbmbox_msg(mbox_msg_t & i_mbox_msg)
 
 void MailboxSp::handle_hbmbox_resp(mbox_msg_t & i_mbox_msg)
 {
-
     //Response for more DMA buffers
-    if(i_mbox_msg.msg_payload.data[0] != 0)
-    {
-        iv_dmaBuffer.addBuffers
-            (i_mbox_msg.msg_payload.data[0]);
-        iv_dma_pend = false;
-        send_msg(); // send next message on queue
-    }
-    else // This is not really a response from the FSP
-        // This is an echo back from echo server of a req the HB MBOX sent.
-    {
-        // This message should never come from the real FSP, so it must be
-        // from the echo server. Responding to the echo server will
-        // make the message echo back again as if it were a response from the
-        // FSP. Since it's not possible to know which buffers the FSP owns, for
-        // testing purposes play the role of the FSP and assume the FSP owns
-        // all DMA buffers and will return them all.
-        // TODO This should probably be removed when/if the echo server
-        // is no longer used.
-        TRACFCOMP(g_trac_mbox,"FAKEFSP returning all DMA buffers");
-
-        i_mbox_msg.msg_payload.data[0] = 0xFFFFFFFFFFFFFFFF;
-
-        // Since the HB is waiting for DMA buffers and holding up all other
-        // messages, just sneek this one on the front of the queue, disable
-        // the dma_pending flag to just long enough to send this message.
-        // All other values in the msg should be left as is.
-        iv_dma_pend = false;
-        iv_sendq.push_front(i_mbox_msg);
-        send_msg(); // respond
-        iv_dma_pend = true;
-    }
+    iv_dmaBuffer.addBuffers
+        (i_mbox_msg.msg_payload.data[0]);
+
+    iv_dma_pend = false;
+
+    send_msg(); // send next message, if there is one
 }
 
 /**
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 01482d7eb..7893ee265 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -131,7 +131,10 @@ namespace MBOX
                 /**
                  * Default constructor
                  */
-                mbox_msg_t() : msg_id(0), msg_queue_id(0), msg_payload() {}
+                mbox_msg_t() : msg_id(0), msg_queue_id(0), msg_payload()
+                {
+                    memset(&msg_payload,'\0',sizeof(msg_t));
+                }
             };
 
             // Private functions
diff --git a/src/usr/mbox/mbox_dma_buffer.C b/src/usr/mbox/mbox_dma_buffer.C
index 667c922de..f04c54aee 100644
--- a/src/usr/mbox/mbox_dma_buffer.C
+++ b/src/usr/mbox/mbox_dma_buffer.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -39,7 +39,8 @@ extern trace_desc_t * g_trac_mbox;
 
 DmaBuffer::DmaBuffer() :
     iv_head(NULL),
-    iv_dir(makeMask(VmmManager::MBOX_DMA_PAGES))
+    iv_dir(makeMask(VmmManager::MBOX_DMA_PAGES)),
+    iv_dma_req_sent(false)
 {
     iv_head = reinterpret_cast<void*>(VmmManager::MBOX_DMA_ADDR);
     initPhysicalArea(iv_head, iv_phys_head);
diff --git a/src/usr/mbox/mbox_dma_buffer.H b/src/usr/mbox/mbox_dma_buffer.H
index 8566af3ae..56101af4a 100644
--- a/src/usr/mbox/mbox_dma_buffer.H
+++ b/src/usr/mbox/mbox_dma_buffer.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -136,6 +136,26 @@ namespace MBOX
                                     base + (i_address-iv_phys_head));
             }
 
+            /**
+             * Set the state of shutdown dma request sent
+             * @param[in] The state to set [true|false]
+             */
+            ALWAYS_INLINE
+            void setShutdownDmaRequestSent(bool i_state)
+            {
+                iv_dma_req_sent = i_state;
+            }
+
+            /**
+             * Query if the shutdown DMA request has been sent
+             * @return state [true|false]
+             */
+            ALWAYS_INLINE
+            bool shutdownDmaRequestSent()
+            {
+                return iv_dma_req_sent;
+            }
+
         private:
 
             /**
@@ -160,6 +180,7 @@ namespace MBOX
             void * iv_head;            //!< Start of DMA memory
             uint64_t iv_phys_head;     //!< Physical translation of iv_head
             uint64_t iv_dir;           //!< 1 bit per 1k buffer, 1 = available
+            bool iv_dma_req_sent;      //!< Reqest sent to retrieve all buffers
 
     };
 }; // namespace