Multinode SBE Window Support

Code for SP ATTN, COMM, and SBE FFDC areas has been extended to
all chips/nodes. The COMM area base address calculation is now
updated to use the lowest addressable location for the master
node.

Change-Id: Ib02e62958a520c60e4bcc5bd9d4c4257893adbd9
RTC:189038
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/56614
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: ILYA SMIRNOV <ismirno@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 37 insertions, 20 deletions

diff --git a/src/include/usr/runtime/runtime.H b/src/include/usr/runtime/runtime.H
index 3ed9d1fd4..c508cc13f 100644
--- a/src/include/usr/runtime/runtime.H
+++ b/src/include/usr/runtime/runtime.H
@@ -328,10 +328,12 @@ void rediscover_hdat( void );
  *             1. CPU controls SP ATTN areas
  *             2. SP_HOST_UNTRUSTED_COMM_AREA_ADDR
  *             3. SBE FFDC
+ * @param[in] i_commBase The lowest addressable location in the system,
+ *            derived from the HRMOR of the master node.
  *
  * @return errlHndl_t Returns nullptr on success; otherwise errlog
  */
-errlHndl_t openUntrustedSpCommArea();
+errlHndl_t openUntrustedSpCommArea(uint64_t i_commbase);
 
 /**
  * @brief Locates the offset of the HDAT structure as directed by the PAYLOAD
diff --git a/src/usr/isteps/istep21/call_host_start_payload.C b/src/usr/isteps/istep21/call_host_start_payload.C
index f51073803..2ffe8ebce 100644
--- a/src/usr/isteps/istep21/call_host_start_payload.C
+++ b/src/usr/isteps/istep21/call_host_start_payload.C
@@ -76,11 +76,15 @@ namespace ISTEP_21
  *
  * @param[in] Host boot master instance number (logical node number)
  * @param[in] Is this the master HB instance [true|false]
+ * @param[in] The lowest addressable location in the system, derived from the
+ *            HRMOR of the master node.
  *
  * @return errlHndl_t - nullptr if succesful, otherwise a pointer to the error
  *      log.
  */
-errlHndl_t callShutdown ( uint64_t i_hbInstance, bool i_masterIntance );
+errlHndl_t callShutdown ( uint64_t i_hbInstance,
+                          bool i_masterIntance,
+                          const uint64_t i_commBase);
 
 /**
  * @brief This function will send an IPC message to all other HB instances
@@ -309,11 +313,14 @@ void* call_host_start_payload (void *io_pArgs)
         }
 #endif
 
+        // calculate lowest addressable memory location to be used as COMM base
+        uint64_t l_commBase = cpu_spr_value(CPU_SPR_HRMOR) - VMM_HRMOR_OFFSET;
+
         //  - Call shutdown using payload base, and payload entry.
         //      - base/entry will be from system attributes
         //      - this will start the payload (Phyp)
         // NOTE: this call will not return if successful.
-        l_errl = callShutdown(this_node, true);
+        l_errl = callShutdown(this_node, true, l_commBase);
         if(l_errl)
         {
             break;
@@ -345,7 +352,8 @@ void* call_host_start_payload (void *io_pArgs)
 // Call shutdown
 //
 errlHndl_t callShutdown ( uint64_t i_masterInstance,
-                          bool i_isMaster)
+                          bool i_isMaster,
+                          const uint64_t i_commBase)
 {
     errlHndl_t err = nullptr;
     uint64_t payloadBase = 0x0;
@@ -364,7 +372,7 @@ errlHndl_t callShutdown ( uint64_t i_masterInstance,
         if (err)
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       ERR_MRK "call_host_start_payload: Failed SBEIO::closeAllUnsecureMemRegions" );
+                       ERR_MRK "callShutdown: Failed SBEIO::closeAllUnsecureMemRegions" );
             break;
         }
 
@@ -448,21 +456,21 @@ errlHndl_t callShutdown ( uint64_t i_masterInstance,
         TARGETING::SpFunctions spFuncs =
                 sys->getAttr<TARGETING::ATTR_SP_FUNCTIONS>();
 
-        if(i_isMaster)
+        // Open untrusted SP communication area if there is a PAYLOAD
+        // NOTE: Must be after all HDAT processing
+        if( !(TARGETING::is_no_load()) )
         {
-            // Open untrusted SP communication area if there is a PAYLOAD
-            // NOTE: Must be after all HDAT processing
-            if( !(TARGETING::is_no_load()) )
+            err = RUNTIME::openUntrustedSpCommArea(i_commBase);
+            if (err)
             {
-                err = RUNTIME::openUntrustedSpCommArea();
-                if (err)
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                               ERR_MRK"call_host_start_payload: Failed openUntrustedSpCommArea" );
-                    break;
-                }
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           ERR_MRK"callShutdown: Failed openUntrustedSpCommArea" );
+                break;
             }
+        }
 
+        if(i_isMaster)
+        {
             // Notify Fsp with appropriate mailbox message.
             err = notifyFsp( istepModeFlag,
                              spFuncs );
@@ -603,6 +611,10 @@ errlHndl_t broadcastShutdown ( uint64_t i_hbInstance )
 
         KernelIpc::start_payload_data_area.node_count = node_count;
 
+        // calculate lowest addressable memory location from the master node
+        // hrmor value and send to slave nodes
+        const uint64_t l_commBase = cpu_spr_value(CPU_SPR_HRMOR) - VMM_HRMOR_OFFSET;
+
         // send message to all other existing hb instances except this one.
         for(uint64_t drawer = 0; drawer < sizeof(node_map); ++drawer)
         {
@@ -625,6 +637,7 @@ errlHndl_t broadcastShutdown ( uint64_t i_hbInstance )
                     msg_t * msg = msg_allocate();
                     msg->type = IPC::IPC_START_PAYLOAD;
                     msg->data[0] = i_hbInstance;
+                    msg->data[1] = l_commBase;
                     err = MBOX::send(MBOX::HB_IPC_MSGQ, msg, node);
                     if (err)
                     {
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index 91ec6ce30..707fa6fb8 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -39,7 +39,8 @@
 namespace ISTEP_21
 {
     extern errlHndl_t callShutdown ( uint64_t i_hbInstance,
-                                     bool i_masterInstance );
+                                     bool i_masterInstance,
+                                     const uint64_t i_commBase );
 };
 
 trace_desc_t* g_trac_ipc = NULL;
@@ -357,7 +358,8 @@ void IpcSp::msgHandler()
                 if(!err)
                 {
                     //  Function will not return unless error
-                    err = ISTEP_21::callShutdown(msg->data[0],false);
+                    err = ISTEP_21::callShutdown(msg->data[0],false,
+                                                                  msg->data[1]);
                 }
 
                 if(err)
diff --git a/src/usr/runtime/populate_hbruntime.C b/src/usr/runtime/populate_hbruntime.C
index f2f23b31e..aa077ca5c 100644
--- a/src/usr/runtime/populate_hbruntime.C
+++ b/src/usr/runtime/populate_hbruntime.C
@@ -3224,7 +3224,7 @@ errlHndl_t persistent_rwAttrRuntimeCheck( void )
     return l_err;
 } // end persistent_rwAttrRuntimeCheck
 
-errlHndl_t openUntrustedSpCommArea()
+errlHndl_t openUntrustedSpCommArea(const uint64_t i_commBase)
 {
     TRACFCOMP( g_trac_runtime, ENTER_MRK "openUntrustedSpCommArea()");
     errlHndl_t l_err = nullptr;
@@ -3300,7 +3300,7 @@ errlHndl_t openUntrustedSpCommArea()
         if(TARGETING::is_phyp_load())
         {
             l_err = SBEIO::openUnsecureMemRegion(
-                                        RUNTIME::SP_HOST_UNTRUSTED_COMM_AREA_ADDR,
+                                        i_commBase,
                                         RUNTIME::SP_HOST_UNTRUSTED_COMM_AREA_SIZE,
                                         true, //true=Read-Write
                                         l_procChip);