Mailbox DMA buffer capabilities

RTC:34032

Change-Id: Ib1e29210ffc183f9c3bd475ab8d9779b5a448909
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/932
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

10 files changed, 595 insertions, 62 deletions

diff --git a/src/include/kernel/pagemgr.H b/src/include/kernel/pagemgr.H
index aec4470d2..e0d2aa1fa 100644
--- a/src/include/kernel/pagemgr.H
+++ b/src/include/kernel/pagemgr.H
@@ -66,7 +66,7 @@ class PageManager
 
         enum
         {
-            MEMLEN = VmmManager::HTABORG,
+            MEMLEN = VmmManager::MBOX_DMA_ADDR,
             BUCKETS = 16,
         };
 
diff --git a/src/include/kernel/vmmmgr.H b/src/include/kernel/vmmmgr.H
index 7a1fd6f7b..8a8fb6103 100644
--- a/src/include/kernel/vmmmgr.H
+++ b/src/include/kernel/vmmmgr.H
@@ -44,6 +44,11 @@ class VmmManager
             // put the Page Table at the end of our memory space
             PTSIZE = (1 << 18),
             HTABORG = (FULL_MEM_SIZE - PTSIZE),
+
+            // Put the DMA Pages just under the Page Table
+            MBOX_DMA_PAGES = 64,    // must be <= 64
+            MBOX_DMA_PAGESIZE = (1 * KILOBYTE),
+            MBOX_DMA_ADDR = (HTABORG - (MBOX_DMA_PAGES * MBOX_DMA_PAGESIZE))
         };
 
         enum castout_t
diff --git a/src/include/usr/mbox/mbox_queues.H b/src/include/usr/mbox/mbox_queues.H
index 6eadfd32f..dbe1369b0 100644
--- a/src/include/usr/mbox/mbox_queues.H
+++ b/src/include/usr/mbox/mbox_queues.H
@@ -48,11 +48,11 @@ namespace MBOX
         // FSP mailboxes
         FSP_FIRST_MSGQ          = 0x80000000,
         FSP_MAILBOX_MSGQ        = FSP_FIRST_MSGQ,
-        FSP_ECHO_MSGQ,
         FSP_PROGRESS_CODES_MSGQ,
         FSP_TRACE_MSGQ,
         FSP_ERRL_MSGQ,
         // Add FSP services here:
+        FSP_ECHO_MSGQ = 0xFFFFFFFF,     // Fake FSP for test
     };
 
     // Message types are only unique with in each message queue. Each message queue can
@@ -85,6 +85,7 @@ namespace MBOX
         MSG_INVALID_MSG_QUEUE_ID = FIRST_UNSECURE_MSG + 1,
         MSG_INVALID_MSG_TYPE,
         MSG_REQUEST_DMA_BUFFERS,
+        MSG_INITIAL_DMA,
     };
 
 
diff --git a/src/include/usr/mbox/mbox_reasoncodes.H b/src/include/usr/mbox/mbox_reasoncodes.H
index f90870a8d..3229ccc6f 100644
--- a/src/include/usr/mbox/mbox_reasoncodes.H
+++ b/src/include/usr/mbox/mbox_reasoncodes.H
@@ -36,6 +36,8 @@ namespace MBOX
         MOD_MBOXSRV_HNDLR       = 0x04, // MailboxSp::msgHandler
         MOD_MBOXSRV_RCV         = 0x05, // MailboxSp::recv_msg
         MOD_MBOXREGISTER        = 0x06, // MailboxSp::msgq_register
+        MOD_MBOXSRV_FSP_MSG     = 0x07, // MailboxSp::handle_hbmbox_msg
+        MOD_MBOXSRV_SENDMSG     = 0x08, // MailboxSp::send_msg
     };
 
     enum MBOXReasonCode
@@ -53,6 +55,7 @@ namespace MBOX
         RC_PARITY_ERR                = HBMBOX_COMP_ID | 0x0B,
         RC_INVALID_MBOX_MSG_TYPE     = HBMBOX_COMP_ID | 0x0C,
         RC_MBOX_SERVICE_NOT_READY    = HBMBOX_COMP_ID | 0x0D,
+        RC_INVALID_DMA_LENGTH        = HBMBOX_COMP_ID | 0x0E,
     };
 };
 
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index dd5988841..e8b017a70 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -58,6 +58,7 @@ MailboxSp::MailboxSp()
         iv_msgQ(),
         iv_sendq(),
         iv_respondq(),
+        iv_dmaBuffer(),
         iv_trgt(NULL),
         iv_rts(true),
         iv_dma_pend(false)
@@ -100,6 +101,14 @@ errlHndl_t MailboxSp::_init()
         task_create(MailboxSp::msg_handler, NULL);
     }
 
+    // Send message to FSP on base DMA buffer zone
+    msg_t * msg = msg_allocate();
+    msg->type = MSG_INITIAL_DMA;
+    msg->data[0] = 0;
+    msg->data[1] = reinterpret_cast<uint64_t>(iv_dmaBuffer.getDmaBufferHead());
+    msg->extra_data = NULL;
+    MBOX::send(FSP_MAILBOX_MSGQ,msg);
+
     return err;
 }
 
@@ -169,7 +178,24 @@ void MailboxSp::msgHandler()
 
                         if(mbox_status & MBOX_DATA_PENDING)
                         {
-                            recv_msg(mbox_msg);
+                            trace_msg("RECV",mbox_msg);
+                            if(mbox_msg.msg_queue_id == HB_MAILBOX_MSGQ)
+                            {
+                                // msg to hb mailbox from fsp mbox
+                                handle_hbmbox_msg(mbox_msg);
+                            }
+                            else if((mbox_msg.msg_queue_id==FSP_MAILBOX_MSGQ)&&
+                                    (mbox_msg.msg_payload.type ==
+                                     MSG_REQUEST_DMA_BUFFERS))
+                            {
+                                // This is a response from The FSP
+                                handle_hbmbox_resp(mbox_msg);
+                            }
+                            else
+                            {
+                                // anything else
+                                recv_msg(mbox_msg);
+                            }
                         }
 
                         // Look for error status from MB hardware
@@ -407,37 +433,94 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
 
     iv_msg_to_send = *mbox_msg; //copy
 
-    // TODO remove the following line when DMA buffer support is available
-    payload->extra_data = NULL;
-
     // Is a DMA buffer needed?
-    if(payload->extra_data != NULL)
+    if((payload->extra_data != NULL) ||
+       ((iv_msg_to_send.msg_queue_id == HB_MAILBOX_MSGQ) &&
+        (payload->type == MSG_REQUEST_DMA_BUFFERS)))
 
     {
-        size_t size = payload->data[1];
-        void * dma_buffer = NULL;
-        // TODO getDMA buffer
+        uint64_t dma_size = payload->data[1];
+
+        if(payload->extra_data == NULL) // DMA req. from FSP.
+        {
+            dma_size = payload->data[0];
+        }
+
+        // getBuffer() returns bit map in dma_size variable.
+        void * dma_buffer = iv_dmaBuffer.getBuffer(dma_size);
 
         if(dma_buffer)
         {
-            memcpy(dma_buffer,payload->extra_data,size);
-            iv_msg_to_send.msg_payload.extra_data = dma_buffer;
+            if(payload->extra_data != NULL)
+            {
+                memcpy(dma_buffer,payload->extra_data,payload->data[1]);
+                iv_msg_to_send.msg_payload.extra_data = dma_buffer;
 
-            free(payload->extra_data);
+                free(payload->extra_data);
+            }
+            else  // DMA buffer request from FSP
+            {
+                iv_msg_to_send.msg_payload.data[0] = dma_size; // bitmap
+                iv_msg_to_send.msg_payload.data[1] = 
+                    reinterpret_cast<uint64_t>(dma_buffer);
+            }
             iv_sendq.pop_front();
         }
         else // can't get buffer
         {
-            //   -- can't send the current message - leave it on the queue
-            //   -- Instead send a message to FSP for more buffers
-            //   TODO set sync flag when  implementing DMA buffer
-            iv_msg_to_send.msg_id = 0;
-            iv_msg_to_send.msg_queue_id = FSP_MAILBOX_MSGQ;
-            iv_msg_to_send.msg_payload.type = MSG_REQUEST_DMA_BUFFERS;
-            iv_msg_to_send.msg_payload.data[0] = 0;
-            iv_msg_to_send.msg_payload.data[1] = 0;
-            iv_msg_to_send.msg_payload.extra_data = NULL;
-            iv_dma_pend = true;
+            if(!iv_dmaBuffer.ownsAllBlocks())
+            {
+                //   -- can't send the current message - leave it on the queue
+                //   -- Instead send a message to FSP for more buffers
+                iv_msg_to_send.msg_id = 0;
+                iv_msg_to_send.msg_queue_id = FSP_MAILBOX_MSGQ;
+                iv_msg_to_send.msg_payload.type = MSG_REQUEST_DMA_BUFFERS;
+                iv_msg_to_send.msg_payload.data[0] = 0;
+                iv_msg_to_send.msg_payload.data[1] = 0;
+                iv_msg_to_send.msg_payload.extra_data = NULL;
+                iv_msg_to_send.msg_payload.__reserved__async = 1;
+                iv_dma_pend = true;
+            }
+            else
+            {
+                // message is asking from more DMA space than exists
+                TRACFCOMP(g_trac_mbox,
+                          ERR_MRK
+                          "MailboxSp::send_msg - Message dropped. "
+                          "Can't get DMA buffer size %d. queueid 0x%x",
+                          payload->data[1],
+                          iv_msg_to_send.msg_queue_id);
+
+                iv_sendq.pop_front();
+                if(payload->extra_data == NULL)  //Request was from FSP
+                {
+                    // just respond with failure
+                    iv_msg_to_send.msg_payload.data[0] = 0;
+                    iv_msg_to_send.msg_payload.data[1] = 0;
+                }
+                else
+                {
+                    /*@ errorlog tag
+                     * @errortype       ERRL_SEV_INFORMATIONAL
+                     * @moduleid        MOD_MBOXSRV_SENDMSG
+                     * @reasoncode      RC_INVALID_DMA_LENGTH
+                     * @userdata1       DMA length requested
+                     * @userdata2       queue_id
+                     * @defdesc         Failed to allocate a DMA buffer.
+                     *                  Message dropped.
+                     */
+                    err = new ERRORLOG::ErrlEntry
+                        (
+                         ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                         MBOX::MOD_MBOXSRV_SENDMSG,
+                         MBOX::RC_INVALID_DMA_LENGTH,      //  reason Code
+                         payload->data[1],                 // DMA data len
+                         iv_msg_to_send.msg_queue_id       // MSG queueid
+                        );
+
+                    send_msg();  // drop this message, but send next message
+                }
+            }
         }
     }
     else // simple message
@@ -445,18 +528,27 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
         iv_sendq.pop_front();
     }
 
-    size_t mbox_msg_len = sizeof(mbox_msg_t);
-    iv_rts = false;
+    if(!err)
+    {
+        size_t mbox_msg_len = sizeof(mbox_msg_t);
+        iv_rts = false;
 
-    trace_msg("SEND",iv_msg_to_send);
+        trace_msg("SEND",iv_msg_to_send);
 
-    err = DeviceFW::deviceWrite(iv_trgt,
-                                &iv_msg_to_send,
-                                mbox_msg_len,
-                                DeviceFW::MAILBOX);
+        err = DeviceFW::deviceWrite(iv_trgt,
+                                    &iv_msg_to_send,
+                                    mbox_msg_len,
+                                    DeviceFW::MAILBOX);
+    }
 
     if(err)
     {
+        TRACFCOMP(g_trac_mbox,
+                  ERR_MRK
+                  "MBOX send_msg could not send message. queue:%x type:%x",
+                  iv_msg_to_send.msg_queue_id,
+                  iv_msg_to_send.msg_payload.type);
+
         // If the message that could not be sent was a sync msg
         // originating from HB then there is a task waiting for a response.
         if(!msg_is_async(&(iv_msg_to_send.msg_payload)))
@@ -497,18 +589,11 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
     errlHndl_t err = NULL;
     int rc = 0;
 
-    // TODO look for response to requests for more DMA buffers
-    // and handle now
-    // add dma buffers
-    // iv_dma_pend = false;
-    // call send_msg to send next message if available and rts
-    //
+    //trace_msg("RECV",i_mbox_msg);
 
     msg_t * msg = msg_allocate();
     *msg = i_mbox_msg.msg_payload;  // copy
 
-    trace_msg("RECV",i_mbox_msg);
-
     // Handle moving data from DMA buffer
     if(msg->extra_data != NULL)
     {
@@ -516,7 +601,7 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
         void * buf = malloc(msg_sz);
         memcpy(buf,msg->extra_data,msg_sz);
 
-        // DMArelease(msg->extra_data,msg_sz); TODO
+        iv_dmaBuffer.release(msg->extra_data,msg_sz);
         msg->extra_data = buf;
         // receiver of the message frees buffer.
     }
@@ -535,7 +620,11 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
         // remove from the list
         iv_respondq.erase(response);
 
-        *(response->key) = *msg; // copy
+        // resonse->key points to original carrier msg
+        // reponse->key->extra_data points to the orignal msg
+        // Overwrite original message with response
+
+        *(reinterpret_cast<msg_t *>((response->key)->extra_data)) = *msg; // copy
 
         msg_free(msg);
 
@@ -619,7 +708,7 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
                 mbox_msg.msg_payload.data[0] = msg->type;
                 mbox_msg.msg_payload.data[1] = i_mbox_msg.msg_queue_id;
                 mbox_msg.msg_payload.extra_data = NULL;
-                mbox_msg.msg_payload.__reserved__async = 1;
+                mbox_msg.msg_payload.__reserved__async = 0; // async
 
 
                 send_msg(&mbox_msg);
@@ -645,7 +734,7 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
                      MBOX::MOD_MBOXSRV_RCV,
                      MBOX::RC_INVALID_MESSAGE_TYPE  ,    //  reason Code
                      i_mbox_msg.msg_queue_id,            // rc from msg_send
-                     0
+                     i_mbox_msg.msg_payload.type
                     );
 
                 err->collectTrace(HBMBOXMSG_TRACE_NAME);
@@ -655,7 +744,9 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
                 msg_free(msg);
             }
         }
-        else  // unregistered msg_queue_id
+        // Else unregisteredd msg_queue_id
+        //  For NOW, ignore FSP mailbox stuff bounced back by the echo server
+        else if(i_mbox_msg.msg_queue_id != FSP_MAILBOX_MSGQ)
         {
 
             TRACFCOMP(g_trac_mbox,
@@ -670,10 +761,14 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
             mbox_msg.msg_payload.type = MSG_INVALID_MSG_QUEUE_ID;
             mbox_msg.msg_payload.data[0] = i_mbox_msg.msg_queue_id;
             mbox_msg.msg_payload.extra_data = NULL;
-            mbox_msg.msg_payload.__reserved__async = 1;
+            mbox_msg.msg_payload.__reserved__async = 0; // async
 
 
-            send_msg(&mbox_msg);
+            // prevent infinite loop with echo mb server
+            if(i_mbox_msg.msg_queue_id != FSP_MAILBOX_MSGQ)
+            {
+                send_msg(&mbox_msg);
+            }
 
 
             /*@ errorlog tag
@@ -701,7 +796,106 @@ void MailboxSp::recv_msg(mbox_msg_t & i_mbox_msg)
 
             free(msg->extra_data);
             msg_free(msg);
-       }
+        }
+        else // This is a bounce-back msg from the echo server - Ignore
+        {
+            free(msg->extra_data);
+            msg_free(msg);
+        }
+    }
+}
+
+void MailboxSp::handle_hbmbox_msg(mbox_msg_t & i_mbox_msg)
+{
+    if(i_mbox_msg.msg_payload.type == MSG_REQUEST_DMA_BUFFERS)
+    {
+        // DMA req. will be resolved by send_msg
+        send_msg(&i_mbox_msg);   // response message
+    }
+    else if(i_mbox_msg.msg_payload.type == MSG_INVALID_MSG_QUEUE_ID)
+    {
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_INFORMATIONAL
+         * @moduleid        MOD_MBOXSRV_FSP_MSG
+         * @reasoncode      RC_INVALID_QUEUE
+         * @userdata1       msg queue
+         * @defdesc         Message from FSP. A message queue sent to FSP
+         *                  was not within a valid range
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_INFORMATIONAL,
+             MBOX::MOD_MBOXSRV_FSP_MSG,
+             MBOX::RC_INVALID_QUEUE,
+             i_mbox_msg.msg_payload.data[0],
+             0
+            );
+
+        err->collectTrace(HBMBOXMSG_TRACE_NAME);
+
+        errlCommit(err,HBMBOX_COMP_ID);
+    }
+    else
+    {
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_INFORMATIONAL
+         * @moduleid        MOD_MBOXSRV_FSP_MSG
+         * @reasoncode      RC_INVALID_MBOX_MSG_TYPE
+         * @userdata1       msg type
+         * @userdata2       msg queue id
+         * @defdesc         Message from FSP. A message type sent to FSP
+         *                  was not within a valid range
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_INFORMATIONAL,
+             MBOX::MOD_MBOXSRV_FSP_MSG,
+             MBOX::RC_INVALID_MBOX_MSG_TYPE,
+             i_mbox_msg.msg_payload.data[0],
+             i_mbox_msg.msg_payload.data[1]
+            );
+
+        err->collectTrace(HBMBOXMSG_TRACE_NAME);
+
+        errlCommit(err,HBMBOX_COMP_ID);
+    }
+}
+
+
+void MailboxSp::handle_hbmbox_resp(mbox_msg_t & i_mbox_msg)
+{
+
+    //Response for more DMA buffers
+    if(i_mbox_msg.msg_payload.data[0] != 0)
+    {
+        iv_dmaBuffer.addBuffers
+            (i_mbox_msg.msg_payload.data[0]);
+        iv_dma_pend = false;
+        send_msg(); // send next message on queue
+    }
+    else // This is not really a response from the FSP
+        // This is an echo back from echo server of a req the HB MBOX sent.
+    {
+        // This message should never come from the real FSP, so it must be
+        // from the echo server. Responding to the echo server will
+        // make the message echo back again as if it were a response from the
+        // FSP. Since it's not possible to know which buffers the FSP owns, for
+        // testing purposes play the role of the FSP and assume the FSP owns
+        // all DMA buffers and will return them all.
+        // TODO This should probably be removed when/if the echo server
+        // is no longer used.
+        TRACFCOMP(g_trac_mbox,"FAKEFSP returning all DMA buffers");
+
+        i_mbox_msg.msg_payload.data[0] = 0xFFFFFFFFFFFFFFFF;
+
+        // Since the HB is waiting for DMA buffers and holding up all other
+        // messages, just sneek this one on the front of the queue, disable
+        // the dma_pending flag to just long enough to send this message.
+        // All other values in the msg should be left as is.
+        iv_dma_pend = false;
+        iv_sendq.push_front(i_mbox_msg);
+        send_msg(); // respond
+        iv_dma_pend = true;
     }
 }
 
@@ -775,7 +969,7 @@ errlHndl_t MailboxSp::msgq_register(queue_id_t i_queue_id, msg_q_t i_msgQ)
     {
         iv_registry[i_queue_id] = i_msgQ;
 
-        TRACFCOMP(g_trac_mbox,INFO_MRK"MailboxSp::msgq_register queue id %d",
+        TRACFCOMP(g_trac_mbox,INFO_MRK"MailboxSp::msgq_register queue id 0x%x",
                   i_queue_id);
     }
     else
@@ -816,7 +1010,7 @@ msg_q_t MailboxSp::msgq_unregister(queue_id_t i_queue_id)
     {
         msgQ = r->second;
         iv_registry.erase(r);
-        TRACFCOMP(g_trac_mbox,INFO_MRK"MailboxSp::msgq_unregister queue id %d",
+        TRACFCOMP(g_trac_mbox,INFO_MRK"MailboxSp::msgq_unregister queue id 0x%x",
                   i_queue_id);
     }
     return msgQ;
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index 9545dc076..b3c806d28 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -26,6 +26,7 @@
  * @file mailboxrp.C
  * @brief Mailbox service provider declariation
  */
+#include "mbox_dma_buffer.H"
 #include <stdint.h>
 #include <builtins.h>
 #include <mbox/mboxif.H>
@@ -178,6 +179,18 @@ namespace MBOX
             void trace_msg(const char * i_text,
                            const mbox_msg_t & i_mbox_msg) const;
 
+            /**
+             * Handle a message to hbmbox from fspmbox
+             * @param[in] i_mbox_msg, the mbox message
+             */
+            void handle_hbmbox_msg(mbox_msg_t & i_mbox_msg);
+
+            /**
+             * Handle a response from FSP
+             * @param[in] i_mbox_msg, the mbox message
+             */
+            void handle_hbmbox_resp(mbox_msg_t & i_mbox_msg);
+
             enum
             {
                 MAX_RETRY_COUNT = 3,
@@ -218,6 +231,7 @@ namespace MBOX
             respond_q_t iv_respondq;    //!< msg respond pending list
             registry_t  iv_registry;    //!< Registered queue
 
+            DmaBuffer   iv_dmaBuffer;   //!< DMA buffer manager
             TARGETING::Target * iv_trgt;//!< mailbox device driver target
             bool        iv_rts;         //!< ready to send flag
             bool        iv_dma_pend;    //!< Request pending for more DMA buffers
diff --git a/src/usr/mbox/makefile b/src/usr/mbox/makefile
index 455b5e581..456836c0d 100644
--- a/src/usr/mbox/makefile
+++ b/src/usr/mbox/makefile
@@ -23,7 +23,7 @@
 ROOTPATH = ../../..
 MODULE = mbox
 
-OBJS = mboxdd.o mailboxsp.o
+OBJS = mboxdd.o mailboxsp.o mbox_dma_buffer.o
 
 SUBDIRS = test.d
 
diff --git a/src/usr/mbox/mbox_dma_buffer.C b/src/usr/mbox/mbox_dma_buffer.C
new file mode 100644
index 000000000..0becfb168
--- /dev/null
+++ b/src/usr/mbox/mbox_dma_buffer.C
@@ -0,0 +1,160 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/usr/mbox/mbox_dma_buffer.C $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2012
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+#include "mbox_dma_buffer.H"
+#include <stdlib.h>
+#include <assert.h>
+#include <util/align.H>
+#include <trace/interface.H>
+#include <kernel/pagemgr.H>
+
+
+#define ALIGN_DMAPAGE(u) (((u) + (VmmManager::MBOX_DMA_PAGESIZE-1)) & \
+                          ~(VmmManager::MBOX_DMA_PAGESIZE-1))
+
+using namespace MBOX;
+
+// Defined in mboxdd.C
+extern trace_desc_t * g_trac_mbox;
+
+DmaBuffer::DmaBuffer() :
+    iv_head(NULL),
+    iv_dir(makeMask(VmmManager::MBOX_DMA_PAGES))
+{
+    iv_head = reinterpret_cast<void*>(VmmManager::MBOX_DMA_ADDR);
+}
+
+
+DmaBuffer::~DmaBuffer()
+{
+}
+
+void DmaBuffer::release(void * i_buffer, size_t i_size)
+{
+    if(!i_buffer) 
+    {
+        return;
+    }
+
+    // Make sure this buffer falls inside the DMA space
+    // If not then it's not a DMA buffer - exit.
+    if(i_buffer < iv_head || 
+       i_buffer >= (static_cast<uint8_t*>(iv_head) + 
+                    (VmmManager::MBOX_DMA_PAGES * 
+                     VmmManager::MBOX_DMA_PAGESIZE)))
+    {
+        TRACDCOMP(g_trac_mbox,
+                  ERR_MRK"MBOX DMA buffer address %p out of range",
+                  i_buffer
+                  );
+        return;
+    }
+
+    // Calculate the # of chunks
+    size_t chunks = ALIGN_DMAPAGE(i_size)/VmmManager::MBOX_DMA_PAGESIZE;
+
+    uint64_t offset =
+        (static_cast<uint8_t *>(i_buffer) - static_cast<uint8_t *>(iv_head)) /
+        VmmManager::MBOX_DMA_PAGESIZE;
+
+    // makeMask will assert if chunks > total possible # of chunks in the dir
+    uint64_t mask = makeMask(chunks);
+
+    mask >>= offset;
+
+    iv_dir |= mask;
+    TRACDCOMP(g_trac_mbox,"MBOX DMA free dir: %016lx",iv_dir);
+}
+
+
+void DmaBuffer::addBuffers(uint64_t i_map)
+{
+    iv_dir |= i_map;
+    TRACDCOMP(g_trac_mbox,"MBOXDMA addBuffers. dir: %016lx",iv_dir);
+}
+
+
+void * DmaBuffer::getBuffer(uint64_t & io_size)
+{
+    // Note: Due to this check, makeMask() will never assert while trying to
+    // make a mask for the requested size. Instead, getBuffer will just return
+    // NULL if the requested size is larger than the total possible DMA buffer
+    // space.
+    if(io_size == 0 || 
+       io_size > (VmmManager::MBOX_DMA_PAGES * VmmManager::MBOX_DMA_PAGESIZE))
+    {
+        io_size = 0;
+        return NULL;
+    }
+
+    void * r_addr = NULL;
+
+    size_t chunks = ALIGN_DMAPAGE(io_size)/VmmManager::MBOX_DMA_PAGESIZE;
+    size_t start_page = 0;
+    size_t shift_count = (MAX_MASK_SIZE + 1) - chunks;
+    uint64_t mask = makeMask(chunks);
+
+    io_size = 0;
+
+    // look for a contiguous block of DMA space.
+    // If shift_count goes to zero, the request could not be granted.
+    while(shift_count)
+    {
+        if((mask & iv_dir) == mask)
+        {
+            break;
+        }
+        mask >>= 1;
+        ++start_page;
+        --shift_count;
+    }
+
+    if(shift_count)
+    {
+        iv_dir &= ~mask;
+        io_size = mask;
+        uint64_t offset = start_page * VmmManager::MBOX_DMA_PAGESIZE;
+        r_addr = static_cast<void*>(static_cast<uint8_t*>(iv_head) + offset);
+    }
+    TRACDCOMP(g_trac_mbox,"MBOX DMA allocate dir: %016lx",iv_dir);
+
+    return r_addr;
+}
+
+uint64_t DmaBuffer::makeMask(uint64_t i_size)
+{
+    assert(i_size <= MAX_MASK_SIZE);
+    uint64_t mask = 0;
+
+    // For some reason (1ul << 64) returns 1, not zero 
+    // The math function pow() converts things to float then back again - bad
+    if(i_size < MAX_MASK_SIZE)
+    {
+        mask = 1ul << i_size;
+    }
+    mask -= 1;
+
+    mask <<= MAX_MASK_SIZE - i_size;
+
+    return mask;
+}
+
diff --git a/src/usr/mbox/mbox_dma_buffer.H b/src/usr/mbox/mbox_dma_buffer.H
new file mode 100644
index 000000000..a05ace6f2
--- /dev/null
+++ b/src/usr/mbox/mbox_dma_buffer.H
@@ -0,0 +1,116 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/usr/mbox/mbox_dma_buffer.H $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2012
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+#if !defined(__MBOX_DMA_BUFFER_H)
+#define __MBOX_DMA_BUFFER_H
+
+/** @file mbox_dma_buffer.H
+ *  @brief Provides the interfaces to the MBOX DMA buffer
+ */
+
+#include <stdint.h>
+#include <builtins.h>
+#include <limits.h>
+#include <kernel/vmmmgr.H>
+
+
+namespace MBOX
+{
+
+    class DmaBuffer
+    {
+        public:
+
+            DmaBuffer();
+            ~DmaBuffer();
+            
+            /**
+             * Get DMA buffer space
+             * @param[in/out] io_size, size in bytes of space needed,
+             *                         0 means get all available buffers
+             *                         bit map is returned showing blocks 
+             *                         aquired.
+             *
+             * @note: There are 64 total buffers in the shared DMA pool. The
+             * bit map has one bit for each buffer. The bit map returned will
+             * have bits on for the buffers who's ownership is being abdicated
+             * by this DMA buffer pool.
+             *
+             * @returns void * to the address allowcated.
+             */
+            void * getBuffer(uint64_t & io_size);
+
+            /**
+             * Release DMA buffer(s) back to the DMA buffer pool
+             * @param[in] i_buffer, pointer to the buffer
+             * @param[in] i_size,  size in bytes of buffer
+             */
+            void release(void * i_buffer, size_t i_size);
+
+            /**
+             * Add DMA buffers to the buffer pool
+             * @param[in] i_map, bit map representing the buffers being added.
+             *
+             * @note: There are 64 total buffers in the shared DMA pool. The
+             * bit map has one bit for each buffer. The bit map given will
+             * have bits on for the buffers who's ownership is being added
+             * to the HB DMA pool.
+             */
+            void addBuffers(uint64_t i_map);
+
+            /**
+             * Get DMA buffer start address
+             * @return DMA buffer start address
+             */
+            ALWAYS_INLINE
+            void * getDmaBufferHead() { return iv_head; }
+
+            /**
+             * Query if all the the DMA blocks are available
+             * @return [true|false]
+             */
+            ALWAYS_INLINE
+            bool ownsAllBlocks() { return iv_dir == makeMask(VmmManager::MBOX_DMA_PAGES); }
+
+
+        private:
+
+            /**
+             * Create the bit mask for the size in DMA_PAGES
+             * @param[in] i_size The size in DMA_PAGES
+             * @post will assert if i_size > MAX_MASK_SIZE
+             * @return The mask - left justified
+             */
+            static uint64_t makeMask(uint64_t i_size);
+
+            enum
+            {
+                MAX_MASK_SIZE = sizeof(uint64_t) * 8,
+            };
+
+            void * iv_head;     //!< Start of DMA memory
+            uint64_t iv_dir;    //!< 1 bit per 1k buffer, 1 = available
+
+    };
+}; // namespace
+
+#endif
diff --git a/src/usr/mbox/test/mboxsptest.H b/src/usr/mbox/test/mboxsptest.H
index e001e19e7..161f7d446 100644
--- a/src/usr/mbox/test/mboxsptest.H
+++ b/src/usr/mbox/test/mboxsptest.H
@@ -67,23 +67,35 @@ class MboxSPTest : public CxxTest::TestSuite
               errlCommit(err,HBMBOX_COMP_ID);
           }
 
-          // Send some messages
-          for(size_t i = 1; i < 2; ++i)
+          // Send some messages - DMA size will force a request
+          // to be sent to FSP for more buffers.
+          for(size_t i = 1; i < 3; ++i)
           {
               msg_t* msg = msg_allocate();
               msg->type = 1;
-              msg->data[1] = 0xFFFFFFFFFFFFFFFFul;
-              msg->extra_data = NULL;
+              msg->data[1] = 33 * 1024;
+              msg->extra_data = malloc(33 * 1024);
+              memcpy(msg->extra_data,"Message junk",13);
 
               msg->data[0] = i;
-              MBOX::send(MBOX::HB_TEST_MSGQ,msg);
+              err = MBOX::send(MBOX::HB_TEST_MSGQ,msg);
+              if(err)
+              {
+                  TS_FAIL("MBOX::send returned an error log");
+                  errlCommit(err,HBMBOX_COMP_ID);
+              }
           }
 
           // Send last message
           msg_t * msg = msg_allocate();
           msg->type = 3;        // use this to terminate while loop below
           msg->extra_data = NULL;
-          MBOX::send(MBOX::HB_TEST_MSGQ,msg);
+          err = MBOX::send(MBOX::HB_TEST_MSGQ,msg);
+          if(err)
+          {
+              TS_FAIL("MBOX::send returned an error log");
+              errlCommit(err,HBMBOX_COMP_ID);
+          }
 
           // now get the messages - they will look like async messages
           // from FSP, even though they are just the echo of the
@@ -101,19 +113,25 @@ class MboxSPTest : public CxxTest::TestSuite
 
               ++msg_idx;
 
-              if(msg->type != 1 ||
-                 msg->data[0] != msg_idx ||
-                 msg->data[1] != 0xFFFFFFFFFFFFFFFFul)
+              if(msg->type != 1 || 
+                 msg->data[0] != msg_idx || 
+                 msg->data[1] != 33 * 1024)
               {
-                  TS_FAIL("MBOX: Unexpected message from FSP");
+                  TS_FAIL("MBOXTEST: Unexpected message from FSP");
 
                   TRACFCOMP(g_trac_mbox,
-                            "MSG from FSP: %d %lx %lx %p",
+                            "MBOXTEST MSG from FSP: %d %lx %lx %p",
                             msg->type,
                             msg->data[0],
                             msg->data[1],
                             msg->extra_data);
               }
+              if(msg->extra_data)
+              {
+                  TRACFCOMP(g_trac_mbox,"MBOXTEST Extra data: %s",
+                            static_cast<char *>(msg->extra_data));
+                  ::free(msg->extra_data);
+              }
 
               msg_free(msg);
           }
@@ -140,7 +158,13 @@ class MboxSPTest : public CxxTest::TestSuite
           msg->data[1] = 1;
           msg->extra_data = NULL;
 
-          MBOX::sendrecv(MBOX::FSP_ECHO_MSGQ,msg);
+          errlHndl_t err = MBOX::sendrecv(MBOX::FSP_ECHO_MSGQ,msg);
+
+          if(err)
+          {
+              TS_FAIL("MBOX::sendrecv returned an error log %p",err);
+              errlCommit(err,HBMBOX_COMP_ID);
+          }
 
           // TODO eventually the return data will be inverted or modified in
           //      some way.
@@ -151,12 +175,28 @@ class MboxSPTest : public CxxTest::TestSuite
               TS_FAIL("Unexpected mailbox sync message returned");
 
               TRACFCOMP(g_trac_mbox,
-                        "SYNC response: %d %lx %lx %p",
+                        "MBOXTEST SYNC response: %d %lx %lx %p",
                         msg->type,
                         msg->data[0],
                         msg->data[1],
                         msg->extra_data);
           }
+
+          msg->type = 2;
+          msg->data[1] = 128 * 1024;  // too big of message
+          msg->extra_data = malloc(8);  
+          err = MBOX::sendrecv(MBOX::FSP_ECHO_MSGQ,msg);
+
+          if(!err)
+          {
+              TS_FAIL("MBOX::sendrecv should return an error log, extra_data too big");
+          }
+          else
+          {
+              delete err;
+          }
+          free(msg->extra_data);
+
           msg_free(msg);
       }
 };