Add mutex to protect queue from SMP effects.

The MailboxSp sendQ was not SMP safe.
The MailboxSp msqQ and response Q are SMP safe.

Change-Id: I4e6deac6592b72fcae4653eee60ea3f07f6568db
CQ: SW405184
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/48570
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 35 insertions, 4 deletions

diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 46bf65297..7260d129b 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -95,6 +95,7 @@ MailboxSp::MailboxSp()
         iv_reclaim_sent_cnt(0),
         iv_reclaim_rsp_cnt(0)
 {
+    mutex_init(&iv_sendq_mutex);
     // mailbox target
     TARGETING::targetService().masterProcChipTargetHandle(iv_trgt);
 }
@@ -631,7 +632,9 @@ void MailboxSp::handleNewMessage(msg_t * i_msg)
         }
         else
         {
+            mutex_lock(&iv_sendq_mutex);
             iv_sendq.push_back(mbox_msg);
+            mutex_unlock(&iv_sendq_mutex);
             TRACFCOMP(g_trac_mbox,"I>Mailbox suspending or suspended.");
             trace_msg("QUEUED",mbox_msg);
         }
@@ -642,6 +645,7 @@ void MailboxSp::handleNewMessage(msg_t * i_msg)
 // Note: When called due to an ACK or retry, iv_rts should be true.
 void MailboxSp::send_msg(mbox_msg_t * i_msg)
 {
+    mutex_lock(&iv_sendq_mutex);
     if(i_msg)
     {
         iv_sendq.push_back(*i_msg);
@@ -654,6 +658,7 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
     //
     if(!iv_rts || iv_dma_pend || iv_sendq.size() == 0)
     {
+        mutex_unlock(&iv_sendq_mutex);
         return;
     }
 
@@ -720,7 +725,7 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
 
                 // track the msg until completion
                 //  actual msg send happens below
-                iv_reclaim_sent_cnt++;
+                __sync_fetch_and_add( &iv_reclaim_sent_cnt, 1 );
             }
             else
             {
@@ -831,6 +836,9 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
             err = NULL;
         }
     }
+
+    mutex_unlock(&iv_sendq_mutex);
+    return;
 }
 
 
@@ -1188,7 +1196,7 @@ void MailboxSp::handle_hbmbox_resp(mbox_msg_t & i_mbox_msg)
         (i_mbox_msg.msg_payload.data[0]);
 
     // track response received
-    iv_reclaim_rsp_cnt++;
+    __sync_fetch_and_add( &iv_reclaim_rsp_cnt, 1);
 
     iv_dma_pend = false;
 
@@ -1393,7 +1401,7 @@ void MailboxSp::sendReclaimDmaBfrsMsg( mbox_msg_t & i_mbox_msg )
     i_mbox_msg.msg_payload.__reserved__async = 1;
 
     // track the msg until completion;
-    iv_reclaim_sent_cnt++;
+    __sync_fetch_and_add( &iv_reclaim_sent_cnt, 1 );
 
     send_msg(&i_mbox_msg);
 
@@ -1901,6 +1909,7 @@ void MailboxSp::handleUnclaimed()
 void MailboxSp::handleShutdown()
 {
     // Shutdown the hardware
+    iv_dmaBuffer.clrShutdownDmaRequestSentCnt();
     errlHndl_t err = mboxddShutDown(iv_trgt);
 
 #if (0) // @todo RTC:126643
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 770218342..22fdf45e8 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -359,8 +359,9 @@ namespace MBOX
             typedef std::pair<uint64_t, uint64_t> addr_size_t;
             typedef std::list<addr_size_t> addr_list_t;
 
-            msg_q_t     iv_msgQ;        //!< mailbox mesage queue
+            msg_q_t     iv_msgQ;        //!< mailbox message queue
             send_q_t    iv_sendq;       //!< msg to send queue
+            mutable mutex_t iv_sendq_mutex; // SMP protection
             mbox_msg_t  iv_msg_to_send; //!< message being sent
             respond_q_t iv_respondq;    //!< msg respond pending list
             registry_t  iv_registry;    //!< Registered queue
diff --git a/src/usr/mbox/mbox_dma_buffer.C b/src/usr/mbox/mbox_dma_buffer.C
index 78a4d1f6a..11a3e0af1 100644
--- a/src/usr/mbox/mbox_dma_buffer.C
+++ b/src/usr/mbox/mbox_dma_buffer.C
@@ -49,6 +49,8 @@ DmaBuffer::DmaBuffer() :
 
     iv_phys_head = mm_virt_to_phys(iv_head);
     memset(iv_head, '\0', VmmManager::MBOX_DMA_SIZE);
+
+    mutex_init(&iv_mutex);
 }
 
 
@@ -89,14 +91,20 @@ void DmaBuffer::release(void * i_buffer, size_t i_size)
 
     mask >>= offset;
 
+    mutex_lock(&iv_mutex);
     iv_dir |= mask;
+    mutex_unlock(&iv_mutex);
+
     TRACDCOMP(g_trac_mbox,"MBOX DMA free dir: %016lx",iv_dir);
 }
 
 
 void DmaBuffer::addBuffers(uint64_t i_map)
 {
+    mutex_lock(&iv_mutex);
     iv_dir |= i_map;
+    mutex_unlock(&iv_mutex);
+
     TRACDCOMP(g_trac_mbox,"MBOXDMA addBuffers. dir: %016lx",iv_dir);
 }
 
@@ -123,6 +131,7 @@ void * DmaBuffer::getBuffer(uint64_t & io_size)
 
     io_size = 0;
 
+    mutex_lock(&iv_mutex);
     // look for a contiguous block of DMA space.
     // If shift_count goes to zero, the request could not be granted.
     while(shift_count)
@@ -143,6 +152,8 @@ void * DmaBuffer::getBuffer(uint64_t & io_size)
         uint64_t offset = start_page * VmmManager::MBOX_DMA_PAGESIZE;
         r_addr = static_cast<void*>(static_cast<uint8_t*>(iv_head) + offset);
     }
+    mutex_unlock(&iv_mutex);
+
     TRACDCOMP(g_trac_mbox,"MBOX DMA allocate dir: %016lx",iv_dir);
 
     return r_addr;
diff --git a/src/usr/mbox/mbox_dma_buffer.H b/src/usr/mbox/mbox_dma_buffer.H
index 7ae9c9219..75a48d6f7 100644
--- a/src/usr/mbox/mbox_dma_buffer.H
+++ b/src/usr/mbox/mbox_dma_buffer.H
@@ -148,6 +148,15 @@ namespace MBOX
             }
 
             /**
+             * Clear the count of shutdown dma request sent
+             */
+            ALWAYS_INLINE
+            void clrShutdownDmaRequestSentCnt( void )
+            {
+                iv_dma_req_sent_cnt = 0;
+            }
+
+            /**
              * Query the max number of DMA bfrs
              * @return [max number of DMA bfrs]
              */
@@ -190,6 +199,7 @@ namespace MBOX
             uint64_t iv_dir;           //!< 1 bit per 1k buffer, 1 = available
             int iv_dma_req_sent_cnt;   //!< number of Requests sent to
                                        //    retrieve all buffers
+            mutable mutex_t iv_mutex;  //used to protect get/release
 
     };
 }; // namespace