diff options
| -rw-r--r-- | src/include/usr/mbox/ipc_msg_types.H | 1 | ||||
| -rw-r--r-- | src/include/usr/mbox/mbox_queues.H | 1 | ||||
| -rw-r--r-- | src/usr/isteps/istep21/call_host_runtime_setup.C | 126 | ||||
| -rw-r--r-- | src/usr/mbox/ipcSp.C | 72 | ||||
| -rw-r--r-- | src/usr/util/utiltcemgr.C | 7 |
5 files changed, 201 insertions, 6 deletions
diff --git a/src/include/usr/mbox/ipc_msg_types.H b/src/include/usr/mbox/ipc_msg_types.H index c2d4d623a..5a2ead6c9 100644 --- a/src/include/usr/mbox/ipc_msg_types.H +++ b/src/include/usr/mbox/ipc_msg_types.H @@ -39,6 +39,7 @@ namespace IPC IPC_START_PAYLOAD, IPC_QUERY_CHIPINFO, IPC_SET_SBE_CHIPINFO, + IPC_CLOSE_TCES, }; }; // namespace IPC diff --git a/src/include/usr/mbox/mbox_queues.H b/src/include/usr/mbox/mbox_queues.H index 20f957038..e1e698c91 100644 --- a/src/include/usr/mbox/mbox_queues.H +++ b/src/include/usr/mbox/mbox_queues.H @@ -54,6 +54,7 @@ namespace MBOX HB_POP_ATTR_MSGQ = 10, // populate Attribute response HB_COALESCE_MSGQ = 11, //host_coalesce response HB_SBE_SYSCONFIG_MSGQ = 12, //For SBE System Config response + HB_CLOSE_TCES_MSGQ = 13, // close/disable TCEs // Add HB mbox msg queue ids (services) before this line HB_LAST_VALID_MSGQ, // end of valid HB mbox msgQ ids diff --git a/src/usr/isteps/istep21/call_host_runtime_setup.C b/src/usr/isteps/istep21/call_host_runtime_setup.C index 12b7e4bae..cbab56501 100644 --- a/src/usr/isteps/istep21/call_host_runtime_setup.C +++ b/src/usr/isteps/istep21/call_host_runtime_setup.C @@ -36,10 +36,13 @@ #include <runtime/runtime.H> #include <runtime/customize_attrs_for_payload.H> #include <targeting/common/util.H> +#include <targeting/targplatutil.H> #include <vpd/vpd_if.H> #include <util/utiltce.H> #include <util/utilmclmgr.H> #include <map> +#include <sys/internode.h> +#include <mbox/ipc_msg_types.H> #include <secureboot/service.H> #include <secureboot/containerheader.H> @@ -63,6 +66,117 @@ using namespace TARGETING; namespace ISTEP_21 { + +// Direct non-master nodes to close and disable their TCEs +errlHndl_t closeNonMasterTces(void) +{ + errlHndl_t l_err = nullptr; + uint64_t nodeid = TARGETING::UTIL::getCurrentNodePhysId(); + + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + ENTER_MRK"closeNonMasterTces(): nodeid=%d", nodeid); + + // keep track of the number of messages we send so we + // know how many responses to expect + uint64_t msg_count = 0; + + do{ + + TARGETING::Target * sys = nullptr; + TARGETING::targetService().getTopLevelTarget( sys ); + assert(sys != nullptr, "closeNonMasterTces() system target is nullptr"); + + TARGETING::ATTR_HB_EXISTING_IMAGE_type hb_images = + sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>(); + // This msgQ catches the node responses from the commands + msg_q_t msgQ = msg_q_create(); + l_err = MBOX::msgq_register(MBOX::HB_CLOSE_TCES_MSGQ,msgQ); + + if(l_err) + { + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): MBOX::msgq_register failed!" ); + break; + } + + // loop thru rest all nodes -- sending msg to each + TARGETING::ATTR_HB_EXISTING_IMAGE_type mask = 0x1 << + ((sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8) -1); + + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): HB_EXISTING_IMAGE (mask) = 0x%X, " + "(hb_images=0x%X)", + mask, hb_images); + + for (uint64_t l_node=0; (l_node < MAX_NODES_PER_SYS); l_node++ ) + { + if (l_node == nodeid) + { + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): don't send IPC_CLOSE_TCES " + "message to master node %d", + nodeid ); + continue; + } + + if( 0 != ((mask >> l_node) & hb_images ) ) + { + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): send IPC_CLOSE_TCES message " + "to node %d", + l_node ); + + msg_t * msg = msg_allocate(); + msg->type = IPC::IPC_CLOSE_TCES; + msg->data[0] = l_node; // destination node + msg->data[1] = nodeid; // respond to this node + + // send the message to the slave hb instance + l_err = MBOX::send(MBOX::HB_IPC_MSGQ, msg, l_node); + + if( l_err ) + { + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): MBOX::send to node %d " + "failed", + l_node); + break; + } + + ++msg_count; + + } // end of node to process + } // end for loop on nodes + + // wait for a response to each message we sent + if( l_err == nullptr ) + { + //$TODO RTC:189356 - need timeout here + while(msg_count) + { + msg_t * response = msg_wait(msgQ); + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "closeNonMasterTces(): IPC_CLOSE_TCES : node %d " + "completed", + response->data[0]); + msg_free(response); + --msg_count; + } + } + + MBOX::msgq_unregister(MBOX::HB_CLOSE_TCES_MSGQ); + msg_q_destroy(msgQ); + + } while(0); + + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + EXIT_MRK"closeNonMasterTces(): l_err rc = 0x%X, msg_count=%d", + ERRL_GETRC_SAFE(l_err), msg_count ); + + return l_err; +} + + // Verify PAYLOAD and Move PAYLOAD+HDAT from Temporary TCE-related // memory region to the proper location errlHndl_t verifyAndMovePayload(void) @@ -446,7 +560,6 @@ void* call_host_runtime_setup (void *io_pArgs) // Close PAYLOAD TCEs if (TCE::utilUseTcesForDmas()) { - l_err = TCE::utilClosePayloadTces(); if ( l_err ) { @@ -456,8 +569,15 @@ void* call_host_runtime_setup (void *io_pArgs) break; } - // @TODO RTC 187335 Close TCEs on non-master nodes - + // Close TCEs on non-master nodes + l_err = closeNonMasterTces(); + if ( l_err ) + { + TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, + "Failed closeNonMasterTces" ); + // break from do loop if error occurred + break; + } } // Need to load up the runtime module if it isn't already loaded diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C index 5878a1bf1..d2c824f03 100644 --- a/src/usr/mbox/ipcSp.C +++ b/src/usr/mbox/ipcSp.C @@ -32,8 +32,9 @@ #include <mbox/mbox_reasoncodes.H> #include <intr/interrupt.H> #include <initservice/initserviceif.H> -#include <sbeio/sbeioif.H> - +#include <sbeio/sbeioif.H> +#include <util/utiltce.H> +#include <targeting/targplatutil.H> namespace ISTEP_21 { @@ -321,6 +322,73 @@ void IpcSp::msgHandler() break; } + + case IPC_CLOSE_TCES: + { + TRACFCOMP( g_trac_ipc, + "Closing TCEs on this node (%d)", + TARGETING::UTIL::getCurrentNodePhysId()); + + // make sure util module is loaded + const char * libutil = "libutil.so"; + if ( !VFS::module_is_loaded( libutil ) ) + { + err = VFS::module_load( libutil ); + + if ( err ) + { + TRACFCOMP( g_trac_ipc, + "Could not load util module" ); + } + else + { + mod_loaded = true; + } + } + + if(!err) + { + err = TCE::utilClosePayloadTces(); + } + + if(!err) + { + err = TCE::utilDisableTces(); + } + + if(err) + { + uint32_t l_errPlid = err->plid(); + errlCommit(err, IPC_COMP_ID); + INITSERVICE::doShutdown(l_errPlid, true); + } + + if(mod_loaded) + { + err = VFS::module_unload( libutil ); + + if (err) + { + errlCommit(err, IPC_COMP_ID); + } + else + { + mod_loaded = false; + } + } + + // Respond + err = MBOX::send(MBOX::HB_CLOSE_TCES_MSGQ, msg, msg->data[1] ); + if (err) + { + uint32_t l_errPlid = err->plid(); + errlCommit(err,IPC_COMP_ID); + INITSERVICE::doShutdown(l_errPlid, true); + } + + break; + } + default: TRACFCOMP( g_trac_ipc, diff --git a/src/usr/util/utiltcemgr.C b/src/usr/util/utiltcemgr.C index dc8158751..d925c332a 100644 --- a/src/usr/util/utiltcemgr.C +++ b/src/usr/util/utiltcemgr.C @@ -253,7 +253,12 @@ errlHndl_t utilClosePayloadTces(void) // Close the Unsecure Memory Region that was opened for the FSP to run // PSI Diagnostics Test using the PAYLOAD section // -- addr is a constant for PAYLOAD - errl = SBEIO::closeUnsecureMemRegion(MCL_TMP_ADDR, + // -- Address must be HRMOR-specific + uint64_t hrmorVal = cpu_spr_value(CPU_SPR_HRMOR); + uint64_t addr = hrmorVal - VMM_HRMOR_OFFSET + MCL_TMP_ADDR; + TRACUCOMP(g_trac_tce,"utilClosePayloadTces(): addr=0x%.16llX, hrmor=0x%.16llX", addr, hrmorVal); + + errl = SBEIO::closeUnsecureMemRegion(addr, nullptr); //Master Processor if(errl) { |
