Fixes for multinode istep 21.1

 -Reordered start_payload to ensure that common things like
    disabling special wakeup, closing down SBE windows, core xstop,
    and clearing PM bars are executed as part of our shutdown path
 -Due to above, needed to correct shared libs loaded on non master
    HB instance
 -Added back in the P8 code to pass the payload base to slave HB
    Instances on populate attr IPC message

Change-Id: I5371b9daf7a4e1aadd1743badcd4219604a38ce7
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/53343
Tested-by: William G. Hoffa <wghoffa@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

3 files changed, 89 insertions, 67 deletions

diff --git a/src/usr/isteps/istep21/call_host_start_payload.C b/src/usr/isteps/istep21/call_host_start_payload.C
index 489516f14..7024fd98c 100644
--- a/src/usr/isteps/istep21/call_host_start_payload.C
+++ b/src/usr/isteps/istep21/call_host_start_payload.C
@@ -316,34 +316,6 @@ void* call_host_start_payload (void *io_pArgs)
         }
 #endif
 
-        l_errl = disableSpecialWakeup();
-        if(l_errl)
-        {
-            break;
-        }
-
-        // Tell SBE to Close All Unsecure Memory Regions
-        l_errl = SBEIO::closeAllUnsecureMemRegions();
-        if (l_errl)
-        {
-            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       ERR_MRK "call_host_start_payload: Failed SBEIO::closeAllUnsecureMemRegions" );
-            break;
-        }
-
-        // Open untrusted SP communication area if there is a PAYLOAD
-        // NOTE: Must be after all HDAT processing
-        if( !(TARGETING::is_no_load()) )
-        {
-            l_errl = RUNTIME::openUntrustedSpCommArea();
-            if (l_errl)
-            {
-                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                           ERR_MRK"call_host_start_payload: Failed openUntrustedSpCommArea" );
-                break;
-            }
-        }
-
         //  - Call shutdown using payload base, and payload entry.
         //      - base/entry will be from system attributes
         //      - this will start the payload (Phyp)
@@ -394,35 +366,45 @@ errlHndl_t callShutdown ( uint64_t i_masterInstance,
 
     do
     {
-        if( i_isMaster == false )
+        err = disableSpecialWakeup();
+        if(err)
         {
+            break;
+        }
 
-            // Revert back to standard runtime mode where core checkstops
-            // do not escalate to system checkstops
-            // Workaround for HW286670
+        // Tell SBE to Close All Unsecure Memory Regions
+        err = SBEIO::closeAllUnsecureMemRegions();
+        if (err)
+        {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                    "calling enableCoreCheckstops() in node");
+                       ERR_MRK "call_host_start_payload: Failed SBEIO::closeAllUnsecureMemRegions" );
+            break;
+        }
 
-            err = enableCoreCheckstops();
+        // Revert back to standard runtime mode where core checkstops
+        // do not escalate to system checkstops
+        // Workaround for HW286670
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                   "calling enableCoreCheckstops() in node");
 
-            if ( err )
-            {
-                break;
-            }
+        err = enableCoreCheckstops();
 
-            if(is_phyp_load())
-            {
-                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                        "calling clearPoreBars() in node");
+        if ( err )
+        {
+            break;
+        }
 
-                //If PHYP then clear out the PORE BARs
-                err = clearPoreBars();
-                if( err )
-                {
-                    break;
-                }
-            }
+        if(is_phyp_load())
+        {
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                       "calling clearPoreBars() in node");
 
+            //If PHYP then clear out the PORE BARs
+            err = clearPoreBars();
+            if( err )
+            {
+                break;
+            }
         }
 
         // Get Target Service, and the system target.
@@ -481,6 +463,19 @@ errlHndl_t callShutdown ( uint64_t i_masterInstance,
 
         if(i_isMaster)
         {
+            // Open untrusted SP communication area if there is a PAYLOAD
+            // NOTE: Must be after all HDAT processing
+            if( !(TARGETING::is_no_load()) )
+            {
+                err = RUNTIME::openUntrustedSpCommArea();
+                if (err)
+                {
+                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                               ERR_MRK"call_host_start_payload: Failed openUntrustedSpCommArea" );
+                    break;
+                }
+            }
+
             // Notify Fsp with appropriate mailbox message.
             err = notifyFsp( istepModeFlag,
                              spFuncs );
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index 2004c6f91..f38b92454 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -113,6 +113,10 @@ void IpcSp::msgHandler()
 
                 if(!err)
                 {
+                    // msg->extra_data contains PAYLOAD Base
+                    RUNTIME::setPayloadBaseAddress(
+                        reinterpret_cast<uint64_t>(msg->extra_data));
+
                     // msg->data[0] contains the node number
                     err = RUNTIME::populate_HbRsvMem( msg->data[0] );
                 }
@@ -170,22 +174,32 @@ void IpcSp::msgHandler()
                 break;
 
             case IPC_START_PAYLOAD:
-
-                if ( !VFS::module_is_loaded( "libstart_payload.so" ) )
+            {
+                const int NUM_MOD = 3;
+                const char * mods[NUM_MOD] =
+                   {"libp9_cpuWkup.so", "libistep21.so", "libpm.so"};
+                bool loaded_mods[NUM_MOD] = {false, false, false};
+                for (auto cnt = 0; cnt < NUM_MOD; ++cnt)
                 {
-                    err = VFS::module_load( "libstart_payload.so" );
-
-                    if ( err )
-                    {
-                        TRACFCOMP( g_trac_ipc,
-                                   "Could not load runtime module" );
-                    }
-                    else
+                    if ( !VFS::module_is_loaded( mods[cnt] ) )
                     {
-                        mod_loaded = true;
+                        err = VFS::module_load( mods[cnt] );
+
+                        if ( err )
+                        {
+                            TRACFCOMP( g_trac_ipc,
+                                       "Could not load %s module", mods[cnt] );
+                            break;
+                        }
+                        else
+                        {
+                            loaded_mods[cnt] = true;
+                        }
                     }
                 }
 
+                if (err) break;
+
                 if(!err)
                 {
                     //  Function will not return unless error
@@ -199,21 +213,25 @@ void IpcSp::msgHandler()
                     INITSERVICE::doShutdown(l_errPlid, true);
                 }
 
-                if(mod_loaded)
+                for (auto cnt = 0; cnt < NUM_MOD; ++cnt)
                 {
-                    err = VFS::module_unload( "libstart_payload.so" );
-
-                    if (err)
+                    if ( loaded_mods[cnt] )
                     {
-                        errlCommit(err, IPC_COMP_ID);
+                        err = VFS::module_unload( mods[cnt] );
+
+                        if (err)
+                        {
+                            errlCommit(err, IPC_COMP_ID);
+                        }
+                        loaded_mods[cnt] = false;
                     }
-                    mod_loaded = false;
+
                 }
 
                 msg_free(msg);
 
                 break;
-
+            }
             default:
 
                 TRACFCOMP( g_trac_ipc,
diff --git a/src/usr/runtime/populate_hbruntime.C b/src/usr/runtime/populate_hbruntime.C
index 745be97b9..d8faf94dd 100644
--- a/src/usr/runtime/populate_hbruntime.C
+++ b/src/usr/runtime/populate_hbruntime.C
@@ -2408,6 +2408,7 @@ errlHndl_t populate_hbRuntimeData( void )
         else
         {
             // multi-node system
+            uint64_t payloadBase = sys->getAttr<TARGETING::ATTR_PAYLOAD_BASE>();
 
             // populate our own node specific data + the common stuff
             l_elog = populate_HbRsvMem(nodeid,true);
@@ -2455,6 +2456,7 @@ errlHndl_t populate_hbRuntimeData( void )
                     msg->type = IPC::IPC_POPULATE_ATTRIBUTES;
                     msg->data[0] = l_node;      // destination node
                     msg->data[1] = nodeid;      // respond to this node
+                    msg->extra_data = reinterpret_cast<uint64_t*>(payloadBase);
 
                     // send the message to the slave hb instance
                     l_elog = MBOX::send(MBOX::HB_IPC_MSGQ, msg, l_node);
@@ -2708,5 +2710,12 @@ errlHndl_t openUntrustedSpCommArea()
     return l_err;
 }
 
+void setPayloadBaseAddress(uint64_t i_payloadAddress)
+{
+    TARGETING::Target * sys = NULL;
+    TARGETING::targetService().getTopLevelTarget( sys );
+    sys->setAttr<TARGETING::ATTR_PAYLOAD_BASE>(i_payloadAddress);
+}
+
 } //namespace RUNTIME