Throttle mailbox operations to avoid out-of-memory condition

Change-Id: I18f13ec742efe196e6a2bc3714051e2489567158
RTC: 102852
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/11586
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: Corey V. Swenson <cswenson@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

2 files changed, 210 insertions, 18 deletions

diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 1c293c4ea..e8a9c8865 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -80,7 +80,10 @@ MailboxSp::MailboxSp()
         iv_rts(true),
         iv_dma_pend(false),
         iv_disabled(true),
-        iv_suspended(false)
+        iv_suspended(false),
+        iv_sum_alloc(0),
+        iv_pend_alloc(),
+        iv_allocAddrs()
 {
     // mailbox target
     TARGETING::targetService().masterProcChipTargetHandle(iv_trgt);
@@ -431,6 +434,17 @@ void MailboxSp::msgHandler()
 
                 break;
 
+            case MSG_MBOX_ALLOCATE:
+                handleAllocate(msg);
+                break;
+
+            case MSG_MBOX_DEALLOCATE:
+                {
+                    void * ptr = reinterpret_cast<void*>(msg->data[0]);
+                    deallocate(ptr);
+                }
+                break;
+
             default:
 
                 TRACFCOMP(g_trac_mbox, ERR_MRK "MailboxSp::msgHandler() "
@@ -530,7 +544,7 @@ void MailboxSp::handleNewMessage(msg_t * i_msg)
             TRACDCOMP( g_trac_mbox, "free extra_data %p",
                         mbox_msg.msg_payload.extra_data );
 
-            free ( mbox_msg.msg_payload.extra_data );
+            deallocate ( mbox_msg.msg_payload.extra_data );
 
             mbox_msg.msg_payload.extra_data = NULL;
         }
@@ -621,7 +635,7 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg)
                 iv_msg_to_send.msg_payload.extra_data =
                   reinterpret_cast<void*>(iv_dmaBuffer.toPhysAddr(dma_buffer));
 
-                free(payload->extra_data);
+                deallocate( payload->extra_data );
                 payload->extra_data = NULL;
             }
             else  // DMA buffer request from FSP
@@ -1709,6 +1723,17 @@ void MailboxSp::handleShutdown()
         crit_assert(0);
     }
 
+    if(iv_pend_alloc.size() || iv_allocAddrs.size() || iv_sum_alloc)
+    {
+        TRACFCOMP(g_trac_mbox,WARN_MRK
+                  "Pending memory allocations = %d "
+                  "Allocated memory entrys = %d "
+                  "Allocated memory size = %d",
+                  iv_pend_alloc.size(),
+                  iv_allocAddrs.size(),
+                  iv_sum_alloc);
+    }
+
     msg_respond(iv_msgQ,iv_shutdown_msg);
     TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is shutdown");
 }
@@ -1729,6 +1754,71 @@ void MailboxSp::resume()
     }
 }
 
+void MailboxSp::handleAllocate(msg_t * i_msg)
+{
+    uint64_t size = i_msg->data[0];
+    msg_t * msg = i_msg;
+
+    // Try to allocate storage
+    // If success then respond to message now else respond to message later
+    if((iv_sum_alloc + size) < MAX_ALLOCATION)
+    {
+        uint64_t address = reinterpret_cast<uint64_t>(malloc(size));
+        iv_sum_alloc += size;
+        iv_allocAddrs.push_back(addr_size_t(address,size));
+        msg->data[1] = address;
+        msg_respond(iv_msgQ,msg);
+    }
+    else // save request on pending queue & block task by not responding
+    {
+        iv_pend_alloc.push_back(msg);
+    }
+}
+
+void MailboxSp::deallocate(void * i_ptr)
+{
+    // if MBOX owns this then adjust iv_sum_alloc and
+    // remove addr from iv_allocAddrs
+    addr_list_t::iterator itr = iv_allocAddrs.begin();
+    for(; itr != iv_allocAddrs.end(); ++itr)
+    {
+        if(itr->first == reinterpret_cast<uint64_t>(i_ptr))
+        {
+            break;
+        }
+    }
+
+    free(i_ptr);
+
+    if(itr != iv_allocAddrs.end())
+    {
+        iv_sum_alloc -= itr->second;
+        iv_allocAddrs.erase(itr);
+
+        // check to see if next allocation(s) can be met
+        while(iv_pend_alloc.size() != 0)
+        {
+            msg_t * msg = iv_pend_alloc.front();
+            uint64_t size = msg->data[0];
+            if((iv_sum_alloc + size) < MAX_ALLOCATION)
+            {
+                uint64_t address = reinterpret_cast<uint64_t>(malloc(size));
+                iv_sum_alloc += size;
+                iv_allocAddrs.push_back(addr_size_t(address,size));
+                msg->data[1] = address;
+                msg_respond(iv_msgQ,msg);
+                iv_pend_alloc.pop_front();
+            }
+            else
+            {
+                // stop on first allocation that can't be met
+                // Must keep messages in order
+                break;
+            }
+        }
+    }
+}
+
 // ----------------------------------------------------------------------------
 // External Interfaces @see mboxif.H
 // ----------------------------------------------------------------------------
@@ -2017,21 +2107,22 @@ errlHndl_t MBOX::resume()
 
 errlHndl_t MBOX::makeErrlMsgQSendFail(uint64_t i_errno)
 {
-        TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available");
+    TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available");
 
-        /*@ errorlog tag
-         * @errortype       ERRL_SEV_INFORMATIONAL
-         * @moduleid        MBOX::MOD_MBOX_MSGQ_FAIL
-         * @reasoncode      MBOX::RC_MSG_SEND_ERROR
-         * @userdata1       kernel errno
-         * @userdata2       <unused>
-         *
-         * @devdesc         Message send to mailbox sp failed
-         *
-         */
-        errlHndl_t err = new ERRORLOG::ErrlEntry
-            (
-             ERRORLOG::ERRL_SEV_INFORMATIONAL,    // severity
+    /*@ errorlog tag
+     * @errortype       ERRL_SEV_INFORMATIONAL
+     * @moduleid        MBOX::MOD_MBOX_MSGQ_FAIL
+     * @reasoncode      MBOX::RC_MSG_SEND_ERROR
+     * @userdata1       kernel errno
+     * @userdata2       <unused>
+     *
+     * @devdesc         Message send to mailbox sp failed
+     * @custdesc        A problem occurred during the IPL of the system
+     *
+     */
+    errlHndl_t err = new ERRORLOG::ErrlEntry
+        (
+         ERRORLOG::ERRL_SEV_INFORMATIONAL,    // severity
              MBOX::MOD_MBOX_MSGQ_FAIL,            // moduleid
              MBOX::RC_MSG_SEND_ERROR,             // reason code
              i_errno,
@@ -2041,3 +2132,76 @@ errlHndl_t MBOX::makeErrlMsgQSendFail(uint64_t i_errno)
 
         return err;
 }
+
+// ----------------------------------------------------------------------------
+
+void * MBOX::allocate(size_t i_size)
+{
+    void * response = NULL;
+
+    msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
+    if(mboxQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->type = MSG_MBOX_ALLOCATE;
+        msg->data[0] = i_size;
+
+        int rc = msg_sendrecv(mboxQ, msg);
+
+        if (rc)
+        {
+            TRACFCOMP(g_trac_mbox, ERR_MRK
+                      "MBOX::allocate msg_sendrecv() failed. errno = %d",
+                      rc);
+            // Creating error log would cause recursion on this function.
+            // Leave response NULL and get memory from the heap.
+        }
+        else
+        {
+            response = reinterpret_cast<void *>(msg->data[1]);
+        }
+
+        msg_free(msg);
+    }
+    else
+    {
+        TRACFCOMP(g_trac_mbox, ERR_MRK"MBOX::allocate - "
+                  "Mailbox Service not available");
+    }
+
+    if(NULL == response)
+    {
+        response = malloc(i_size);
+    }
+    
+    return response;
+}
+
+void MBOX::deallocate(void * i_ptr)
+{
+    msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
+    if(mboxQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->type = MSG_MBOX_DEALLOCATE;
+        msg->data[0] = reinterpret_cast<uint64_t>(i_ptr);
+
+        int rc = msg_sendrecv(mboxQ, msg);
+        if(rc)
+        {
+            // kernel problem, Creating error log could cause recursion on
+            // this function - just free memory from heap
+            TRACFCOMP(g_trac_mbox, ERR_MRK
+                      "MBOX::deallocate msg_sendrecv() failed. errno = %d",
+                      rc);
+            free(i_ptr);
+        }
+    }
+    else
+    {
+        TRACFCOMP(g_trac_mbox, ERR_MRK"MBOX::deallocate - "
+                  "Mailbox Service not available");
+        free(i_ptr);
+    }
+}
+
diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H
index b9bcb1b2c..c643fbe6b 100644
--- a/src/usr/mbox/mailboxsp.H
+++ b/src/usr/mbox/mailboxsp.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2014                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -37,6 +39,7 @@
 #include <map>
 #include <util/locked/list.H>
 #include <intr/interrupt.H>
+#include <limits.h>
 
 extern const char* VFS_ROOT_MSG_MBOX;
 
@@ -61,6 +64,8 @@ namespace MBOX
         MSG_MBOX_RESUME,
         MSG_IPC,
         MSG_LOCAL_IPC,
+        MSG_MBOX_ALLOCATE,
+        MSG_MBOX_DEALLOCATE,
     };
 
 
@@ -249,6 +254,20 @@ namespace MBOX
             void invalidMsgResponder(mbox_msg_t & i_mbox_msg);
 
             /**
+             * Handle an allocate message
+             * @param[in] The allocate message
+             */
+            void handleAllocate(msg_t* i_msg);
+
+            /**
+             * Unallocate storage
+             * @param[in] Pointer to allocated storage
+             * @note If i_ptr is not owned by mbox then
+             * storage is just freed
+             */
+            void deallocate(void * i_ptr);
+
+            /**
              * Query Quiesced
              * @returns [true|false]
              */
@@ -258,6 +277,7 @@ namespace MBOX
             enum
             {
                 MAX_RETRY_COUNT = 3,
+                MAX_ALLOCATION = 64 * KILOBYTE,
             };
 
             /**
@@ -288,6 +308,10 @@ namespace MBOX
             typedef std::list<mbox_msg_t> send_q_t;
             typedef Util::Locked::List<msg_respond_t, msg_t *> respond_q_t;
             typedef std::map<queue_id_t,msg_q_t> registry_t;
+            typedef std::list<msg_t *> msg_list_t;
+
+            typedef std::pair<uint64_t, uint64_t> addr_size_t;
+            typedef std::list<addr_size_t> addr_list_t;
 
             msg_q_t     iv_msgQ;        //!< mailbox mesage queue
             send_q_t    iv_sendq;       //!< msg to send queue
@@ -304,6 +328,10 @@ namespace MBOX
             bool        iv_dma_pend;    //!< Request pending for more DMA bufs
             bool        iv_disabled;    //!< Mailboxsp shut off (rejects new)
             bool        iv_suspended;   //!< Mailbox is suspended (queues new)
+            uint64_t    iv_sum_alloc;   //!< Total extra_data storage allocated
+            msg_list_t  iv_pend_alloc;  //!< Pending memory allocations
+            addr_list_t iv_allocAddrs;  //!< memory addresses allocated by mbox
+
     };
 };