HB Mailbox queue messages until destination service is ready.

RTC: 42424

Change-Id: If56ecd8cda845badaf7a8757e2f74eb7d0514398
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1098
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

3 files changed, 47 insertions, 49 deletions

diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index ac7a50c3f..bdb7e895d 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -545,6 +545,8 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
     *msg = i_mbox_msg.msg_payload;  // copy
 
     // Handle moving data from DMA buffer
+    // TODO for pending messages the extra_data is not a DMA address
+    //      so change this to check for valid DMA address instead of NULL.
     if(msg->extra_data != NULL)
     {
         uint64_t msg_sz = msg->data[1];
@@ -698,53 +700,21 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
         //  For NOW, ignore FSP mailbox stuff bounced back by the echo server
         else if(i_mbox_msg.msg_queue_id != FSP_MAILBOX_MSGQ)
         {
+            // Queue message to wait until queue is registered.
+            // TODO - on shutdown, report if iv_pending is not empty.
 
-            TRACFCOMP(g_trac_mbox,
-                      ERR_MRK
-                      "MailboxSp::recv_msg - Message dropped. "
-                      "Unregistered msg queue id 0x%x",
-                      i_mbox_msg.msg_queue_id);
-
-            //  Send a message to the FSP
-            mbox_msg_t mbox_msg;
-            mbox_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
-            mbox_msg.msg_payload.type = MSG_INVALID_MSG_QUEUE_ID;
-            mbox_msg.msg_payload.data[0] = i_mbox_msg.msg_queue_id;
-            mbox_msg.msg_payload.extra_data = NULL;
-            mbox_msg.msg_payload.__reserved__async = 0; // async
+            // copy in non-dma instance of payload msg
+            i_mbox_msg.msg_payload = *msg;
 
+            iv_pendingq.push_back(i_mbox_msg);
 
-            // prevent infinite loop with echo mb server
-            if(i_mbox_msg.msg_queue_id != FSP_MAILBOX_MSGQ)
-            {
-                send_msg(&mbox_msg);
-            }
-
-
-            /*@ errorlog tag
-             * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
-             * @moduleid        MBOX::MOD_MBOXSRV_RCV
-             * @reasoncode      MBOX::RC_UNREGISTERED_MSG_QUEUE
-             * @userdata1       msg queueid
-             * @defdesc         msg queue type is not registered with the
-             *                  mailbox. Message dropped.
-             *
-             */
-            err = new ERRORLOG::ErrlEntry
-                (
-                 ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
-                 MBOX::MOD_MBOXSRV_RCV,
-                 MBOX::RC_UNREGISTERED_MSG_QUEUE,    //  reason Code
-                 i_mbox_msg.msg_queue_id,            // rc from msg_send
-                 0
-                );
-
-            // This trace will have the message content.
-            err->collectTrace(HBMBOXMSG_TRACE_NAME);
+            TRACFCOMP(g_trac_mbox,
+                      INFO_MRK
+                      "MailboxSp::recv_msg. Unregistered msg queue id 0x%x"
+                      " message queued.",
+                      i_mbox_msg.msg_queue_id);
 
-            errlCommit(err,HBMBOX_COMP_ID);
 
-            free(msg->extra_data);
             msg_free(msg);
         }
         else // This is a bounce-back msg from the echo server - Ignore
@@ -921,6 +891,24 @@ errlHndl_t MailboxSp::msgq_register(queue_id_t i_queue_id, msg_q_t i_msgQ)
 
         TRACFCOMP(g_trac_mbox,INFO_MRK"MailboxSp::msgq_register queue id 0x%x",
                   i_queue_id);
+
+        // Look for pending messages and send them
+        // remove_if and remove_copy_if not available
+        size_t size = iv_pendingq.size();
+        while(size--)
+        {
+            send_q_t::iterator mbox_msg = iv_pendingq.begin();
+            if(i_queue_id == mbox_msg->msg_queue_id)
+            {
+                recv_msg(*mbox_msg);
+            }
+            else
+            {
+                iv_pendingq.push_back(*mbox_msg);
+            }
+
+            iv_pendingq.pop_front();
+        }
     }
     else
     {
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index cdcbac331..796a1f36a 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -259,6 +259,8 @@ namespace MBOX
             respond_q_t iv_respondq;    //!< msg respond pending list
             registry_t  iv_registry;    //!< Registered queue
             DmaBuffer   iv_dmaBuffer;   //!< DMA buffer manager
+            send_q_t    iv_pendingq;    //!< Pending for queue registration
+
             TARGETING::Target * iv_trgt;//!< mailbox device driver target
             msg_t *     iv_shutdown_msg;//!< Message to shutdown mbox
             bool        iv_rts;         //!< ready to send flag
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index ad6865d2e..ff43bd5b1 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -36,6 +36,7 @@
 #include <limits.h>
 #include <mbox/mboxif.H>
 #include <targeting/common/util.H>
+#include <sys/time.h>
 
 extern trace_desc_t* g_trac_mbox;
 
@@ -57,17 +58,12 @@ class MboxSPTest : public CxxTest::TestSuite
           {
               return;
           }
+
+          errlHndl_t err = NULL;
           // requires proper simics model
           // msg will get echoed which looks like a new msg from FSP
           // Register a message queue to receive it.
           msg_q_t msgQ = msg_q_create();
-          errlHndl_t err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
-
-          if(err)
-          {
-              TS_FAIL("MBOX: Could not register message queue");
-              errlCommit(err,HBMBOX_COMP_ID);
-          }
 
           // Send some messages - DMA size will force a request
           // to be sent to FSP for more buffers.
@@ -88,6 +84,18 @@ class MboxSPTest : public CxxTest::TestSuite
               }
           }
 
+          nanosleep(0,1000000);
+
+          // late registration to test ability to queue message until
+          // a queue is ready
+          err = MBOX::msgq_register(MBOX::HB_TEST_MSGQ,msgQ);
+          if(err)
+          {
+              TS_FAIL("MBOX: Could not register message queue");
+              errlCommit(err,HBMBOX_COMP_ID);
+          }
+
+
           // Send last message
           msg_t * msg = msg_allocate();
           msg->type = 0xff;        // use this to terminate while loop below