From 9ba94076a3ae491c1d4bd8a2bf7b015f0af17b30 Mon Sep 17 00:00:00 2001 From: Doug Gilbert Date: Wed, 11 Jun 2014 17:05:32 -0500 Subject: Throttle mailbox operations to avoid out-of-memory condition Change-Id: I18f13ec742efe196e6a2bc3714051e2489567158 RTC: 102852 Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/11586 Tested-by: Jenkins Server Reviewed-by: Daniel M. Crowell Reviewed-by: Corey V. Swenson Reviewed-by: A. Patrick Williams III --- src/include/usr/mbox/mbox_reasoncodes.H | 4 +- src/include/usr/mbox/mboxif.H | 25 +++- src/usr/errl/errlmanager.C | 17 ++- src/usr/mbox/mailboxsp.C | 198 +++++++++++++++++++++++++++++--- src/usr/mbox/mailboxsp.H | 30 ++++- src/usr/targeting/attrsync.C | 6 +- src/usr/trace/daemon/daemon.C | 11 +- 7 files changed, 262 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/include/usr/mbox/mbox_reasoncodes.H b/src/include/usr/mbox/mbox_reasoncodes.H index ff5e6a238..6b9e3d8a9 100644 --- a/src/include/usr/mbox/mbox_reasoncodes.H +++ b/src/include/usr/mbox/mbox_reasoncodes.H @@ -5,7 +5,9 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* COPYRIGHT International Business Machines Corp. 2012,2014 */ +/* Contributors Listed Below - COPYRIGHT 2012,2014 */ +/* [+] International Business Machines Corp. */ +/* */ /* */ /* Licensed under the Apache License, Version 2.0 (the "License"); */ /* you may not use this file except in compliance with the License. */ diff --git a/src/include/usr/mbox/mboxif.H b/src/include/usr/mbox/mboxif.H index f9846cbc3..1f56d6c95 100644 --- a/src/include/usr/mbox/mboxif.H +++ b/src/include/usr/mbox/mboxif.H @@ -5,7 +5,9 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* COPYRIGHT International Business Machines Corp. 2012,2014 */ +/* Contributors Listed Below - COPYRIGHT 2012,2014 */ +/* [+] International Business Machines Corp. */ +/* */ /* */ /* Licensed under the Apache License, Version 2.0 (the "License"); */ /* you may not use this file except in compliance with the License. */ @@ -131,6 +133,27 @@ namespace MBOX */ errlHndl_t resume(); + /** + * Allocate storage for mailbox message extra_data for asynchronous + * messages. + * + * @param[in] Number of bytes to allocate + * @return pointer to storage + * + * @note This call may block until memory becomes available if the memory + * sum of of all mbox messages is excessive. + * + * @note a call to MOX::send(..) will release the storage. + */ + void * allocate(size_t); + + /** + * Deallocate storage for a mailbox message extra_data + * + * @param[in] pointer to storage + */ + void deallocate(void * i_ptr); + }; // end namespace MBOX #endif diff --git a/src/usr/errl/errlmanager.C b/src/usr/errl/errlmanager.C index afcf68e87..14a58b05f 100644 --- a/src/usr/errl/errlmanager.C +++ b/src/usr/errl/errlmanager.C @@ -5,7 +5,9 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* COPYRIGHT International Business Machines Corp. 2011,2014 */ +/* Contributors Listed Below - COPYRIGHT 2011,2014 */ +/* [+] International Business Machines Corp. */ +/* */ /* */ /* Licensed under the Apache License, Version 2.0 (the "License"); */ /* you may not use this file except in compliance with the License. */ @@ -530,7 +532,16 @@ msg_t *ErrlManager::sendErrLogToMbox ( errlHndl_t& io_err ) msg->data[0] = io_err->eid(); msg->data[1] = l_msgSize; - void * temp_buff = malloc( l_msgSize ); + void * temp_buff = NULL; + if( iv_isMboxEnabled ) + { + temp_buff = MBOX::allocate( l_msgSize ); + } + else + { + temp_buff = malloc( l_msgSize ); + } + io_err->flatten ( temp_buff, l_msgSize ); msg->extra_data = temp_buff; @@ -551,7 +562,7 @@ msg_t *ErrlManager::sendErrLogToMbox ( errlHndl_t& io_err ) TRACFCOMP(g_trac_errl, ERR_MRK"Failed sending error log to FSP"); //Free the extra data due to the error - free( msg->extra_data ); + MBOX::deallocate( msg->extra_data ); msg_free( msg ); msg = NULL; diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C index 1c293c4ea..e8a9c8865 100644 --- a/src/usr/mbox/mailboxsp.C +++ b/src/usr/mbox/mailboxsp.C @@ -80,7 +80,10 @@ MailboxSp::MailboxSp() iv_rts(true), iv_dma_pend(false), iv_disabled(true), - iv_suspended(false) + iv_suspended(false), + iv_sum_alloc(0), + iv_pend_alloc(), + iv_allocAddrs() { // mailbox target TARGETING::targetService().masterProcChipTargetHandle(iv_trgt); @@ -431,6 +434,17 @@ void MailboxSp::msgHandler() break; + case MSG_MBOX_ALLOCATE: + handleAllocate(msg); + break; + + case MSG_MBOX_DEALLOCATE: + { + void * ptr = reinterpret_cast(msg->data[0]); + deallocate(ptr); + } + break; + default: TRACFCOMP(g_trac_mbox, ERR_MRK "MailboxSp::msgHandler() " @@ -530,7 +544,7 @@ void MailboxSp::handleNewMessage(msg_t * i_msg) TRACDCOMP( g_trac_mbox, "free extra_data %p", mbox_msg.msg_payload.extra_data ); - free ( mbox_msg.msg_payload.extra_data ); + deallocate ( mbox_msg.msg_payload.extra_data ); mbox_msg.msg_payload.extra_data = NULL; } @@ -621,7 +635,7 @@ void MailboxSp::send_msg(mbox_msg_t * i_msg) iv_msg_to_send.msg_payload.extra_data = reinterpret_cast(iv_dmaBuffer.toPhysAddr(dma_buffer)); - free(payload->extra_data); + deallocate( payload->extra_data ); payload->extra_data = NULL; } else // DMA buffer request from FSP @@ -1709,6 +1723,17 @@ void MailboxSp::handleShutdown() crit_assert(0); } + if(iv_pend_alloc.size() || iv_allocAddrs.size() || iv_sum_alloc) + { + TRACFCOMP(g_trac_mbox,WARN_MRK + "Pending memory allocations = %d " + "Allocated memory entrys = %d " + "Allocated memory size = %d", + iv_pend_alloc.size(), + iv_allocAddrs.size(), + iv_sum_alloc); + } + msg_respond(iv_msgQ,iv_shutdown_msg); TRACFCOMP(g_trac_mbox,INFO_MRK"Mailbox is shutdown"); } @@ -1729,6 +1754,71 @@ void MailboxSp::resume() } } +void MailboxSp::handleAllocate(msg_t * i_msg) +{ + uint64_t size = i_msg->data[0]; + msg_t * msg = i_msg; + + // Try to allocate storage + // If success then respond to message now else respond to message later + if((iv_sum_alloc + size) < MAX_ALLOCATION) + { + uint64_t address = reinterpret_cast(malloc(size)); + iv_sum_alloc += size; + iv_allocAddrs.push_back(addr_size_t(address,size)); + msg->data[1] = address; + msg_respond(iv_msgQ,msg); + } + else // save request on pending queue & block task by not responding + { + iv_pend_alloc.push_back(msg); + } +} + +void MailboxSp::deallocate(void * i_ptr) +{ + // if MBOX owns this then adjust iv_sum_alloc and + // remove addr from iv_allocAddrs + addr_list_t::iterator itr = iv_allocAddrs.begin(); + for(; itr != iv_allocAddrs.end(); ++itr) + { + if(itr->first == reinterpret_cast(i_ptr)) + { + break; + } + } + + free(i_ptr); + + if(itr != iv_allocAddrs.end()) + { + iv_sum_alloc -= itr->second; + iv_allocAddrs.erase(itr); + + // check to see if next allocation(s) can be met + while(iv_pend_alloc.size() != 0) + { + msg_t * msg = iv_pend_alloc.front(); + uint64_t size = msg->data[0]; + if((iv_sum_alloc + size) < MAX_ALLOCATION) + { + uint64_t address = reinterpret_cast(malloc(size)); + iv_sum_alloc += size; + iv_allocAddrs.push_back(addr_size_t(address,size)); + msg->data[1] = address; + msg_respond(iv_msgQ,msg); + iv_pend_alloc.pop_front(); + } + else + { + // stop on first allocation that can't be met + // Must keep messages in order + break; + } + } + } +} + // ---------------------------------------------------------------------------- // External Interfaces @see mboxif.H // ---------------------------------------------------------------------------- @@ -2017,21 +2107,22 @@ errlHndl_t MBOX::resume() errlHndl_t MBOX::makeErrlMsgQSendFail(uint64_t i_errno) { - TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available"); + TRACFCOMP(g_trac_mbox, ERR_MRK"Mailbox Service not available"); - /*@ errorlog tag - * @errortype ERRL_SEV_INFORMATIONAL - * @moduleid MBOX::MOD_MBOX_MSGQ_FAIL - * @reasoncode MBOX::RC_MSG_SEND_ERROR - * @userdata1 kernel errno - * @userdata2 - * - * @devdesc Message send to mailbox sp failed - * - */ - errlHndl_t err = new ERRORLOG::ErrlEntry - ( - ERRORLOG::ERRL_SEV_INFORMATIONAL, // severity + /*@ errorlog tag + * @errortype ERRL_SEV_INFORMATIONAL + * @moduleid MBOX::MOD_MBOX_MSGQ_FAIL + * @reasoncode MBOX::RC_MSG_SEND_ERROR + * @userdata1 kernel errno + * @userdata2 + * + * @devdesc Message send to mailbox sp failed + * @custdesc A problem occurred during the IPL of the system + * + */ + errlHndl_t err = new ERRORLOG::ErrlEntry + ( + ERRORLOG::ERRL_SEV_INFORMATIONAL, // severity MBOX::MOD_MBOX_MSGQ_FAIL, // moduleid MBOX::RC_MSG_SEND_ERROR, // reason code i_errno, @@ -2041,3 +2132,76 @@ errlHndl_t MBOX::makeErrlMsgQSendFail(uint64_t i_errno) return err; } + +// ---------------------------------------------------------------------------- + +void * MBOX::allocate(size_t i_size) +{ + void * response = NULL; + + msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX); + if(mboxQ) + { + msg_t * msg = msg_allocate(); + msg->type = MSG_MBOX_ALLOCATE; + msg->data[0] = i_size; + + int rc = msg_sendrecv(mboxQ, msg); + + if (rc) + { + TRACFCOMP(g_trac_mbox, ERR_MRK + "MBOX::allocate msg_sendrecv() failed. errno = %d", + rc); + // Creating error log would cause recursion on this function. + // Leave response NULL and get memory from the heap. + } + else + { + response = reinterpret_cast(msg->data[1]); + } + + msg_free(msg); + } + else + { + TRACFCOMP(g_trac_mbox, ERR_MRK"MBOX::allocate - " + "Mailbox Service not available"); + } + + if(NULL == response) + { + response = malloc(i_size); + } + + return response; +} + +void MBOX::deallocate(void * i_ptr) +{ + msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX); + if(mboxQ) + { + msg_t * msg = msg_allocate(); + msg->type = MSG_MBOX_DEALLOCATE; + msg->data[0] = reinterpret_cast(i_ptr); + + int rc = msg_sendrecv(mboxQ, msg); + if(rc) + { + // kernel problem, Creating error log could cause recursion on + // this function - just free memory from heap + TRACFCOMP(g_trac_mbox, ERR_MRK + "MBOX::deallocate msg_sendrecv() failed. errno = %d", + rc); + free(i_ptr); + } + } + else + { + TRACFCOMP(g_trac_mbox, ERR_MRK"MBOX::deallocate - " + "Mailbox Service not available"); + free(i_ptr); + } +} + diff --git a/src/usr/mbox/mailboxsp.H b/src/usr/mbox/mailboxsp.H index b9bcb1b2c..c643fbe6b 100644 --- a/src/usr/mbox/mailboxsp.H +++ b/src/usr/mbox/mailboxsp.H @@ -5,7 +5,9 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* COPYRIGHT International Business Machines Corp. 2012,2014 */ +/* Contributors Listed Below - COPYRIGHT 2012,2014 */ +/* [+] International Business Machines Corp. */ +/* */ /* */ /* Licensed under the Apache License, Version 2.0 (the "License"); */ /* you may not use this file except in compliance with the License. */ @@ -37,6 +39,7 @@ #include #include #include +#include extern const char* VFS_ROOT_MSG_MBOX; @@ -61,6 +64,8 @@ namespace MBOX MSG_MBOX_RESUME, MSG_IPC, MSG_LOCAL_IPC, + MSG_MBOX_ALLOCATE, + MSG_MBOX_DEALLOCATE, }; @@ -248,6 +253,20 @@ namespace MBOX */ void invalidMsgResponder(mbox_msg_t & i_mbox_msg); + /** + * Handle an allocate message + * @param[in] The allocate message + */ + void handleAllocate(msg_t* i_msg); + + /** + * Unallocate storage + * @param[in] Pointer to allocated storage + * @note If i_ptr is not owned by mbox then + * storage is just freed + */ + void deallocate(void * i_ptr); + /** * Query Quiesced * @returns [true|false] @@ -258,6 +277,7 @@ namespace MBOX enum { MAX_RETRY_COUNT = 3, + MAX_ALLOCATION = 64 * KILOBYTE, }; /** @@ -288,6 +308,10 @@ namespace MBOX typedef std::list send_q_t; typedef Util::Locked::List respond_q_t; typedef std::map registry_t; + typedef std::list msg_list_t; + + typedef std::pair addr_size_t; + typedef std::list addr_list_t; msg_q_t iv_msgQ; //!< mailbox mesage queue send_q_t iv_sendq; //!< msg to send queue @@ -304,6 +328,10 @@ namespace MBOX bool iv_dma_pend; //!< Request pending for more DMA bufs bool iv_disabled; //!< Mailboxsp shut off (rejects new) bool iv_suspended; //!< Mailbox is suspended (queues new) + uint64_t iv_sum_alloc; //!< Total extra_data storage allocated + msg_list_t iv_pend_alloc; //!< Pending memory allocations + addr_list_t iv_allocAddrs; //!< memory addresses allocated by mbox + }; }; diff --git a/src/usr/targeting/attrsync.C b/src/usr/targeting/attrsync.C index d6c6fd952..dbefa328f 100644 --- a/src/usr/targeting/attrsync.C +++ b/src/usr/targeting/attrsync.C @@ -5,7 +5,9 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* COPYRIGHT International Business Machines Corp. 2012,2014 */ +/* Contributors Listed Below - COPYRIGHT 2012,2014 */ +/* [+] International Business Machines Corp. */ +/* */ /* */ /* Licensed under the Apache License, Version 2.0 (the "License"); */ /* you may not use this file except in compliance with the License. */ @@ -107,7 +109,7 @@ namespace TARGETING msg->data[1] = PAGESIZE; // allocated storage will always be 4k - msg->extra_data = malloc( PAGESIZE ); + msg->extra_data = MBOX::allocate( PAGESIZE ); // copy the attributes from mem to our buffer. memcpy( msg->extra_data, diff --git a/src/usr/trace/daemon/daemon.C b/src/usr/trace/daemon/daemon.C index 259d79da3..a51048128 100644 --- a/src/usr/trace/daemon/daemon.C +++ b/src/usr/trace/daemon/daemon.C @@ -456,13 +456,14 @@ namespace TRACEDAEMON msg_t* msg = msg_allocate(); msg->type = DaemonIf::TRACE_CONT_TRACE_BUFFER; msg->data[1] = i_size; - msg->extra_data = i_buffer; + msg->extra_data = MBOX::allocate(i_size); + memcpy(msg->extra_data,i_buffer,i_size); + free(i_buffer); errlHndl_t l_errl = MBOX::send(MBOX::FSP_TRACE_MSGQ, msg); if (l_errl) { errlCommit(l_errl, TRACE_COMP_ID); - free(i_buffer); msg_free(msg); } } @@ -494,7 +495,7 @@ namespace TRACEDAEMON if (l_errl) { errlCommit(l_errl, TRACE_COMP_ID); - free(i_buffer); + MBOX::deallocate(i_buffer); msg_free(msg); } } @@ -730,7 +731,9 @@ namespace TRACEDAEMON // curBuffer pointer is transfered to mailbox now. } - curBuffer = reinterpret_cast(malloc(PAGESIZE)); + curBuffer = reinterpret_cast + (MBOX::allocate(PAGESIZE)); + memset(curBuffer, '\0', PAGESIZE); curBuffer[0] = TRACE_BUF_CONT; curBufferSize = 1; -- cgit v1.2.1