implement start_payload for multi-drawer

RTC: 71994
Change-Id: I422f349d5588731a5e7cfc504d96e497958d6b95
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/7426
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

14 files changed, 371 insertions, 68 deletions

diff --git a/src/include/kernel/ipc.H b/src/include/kernel/ipc.H
index f411808c3..2350b6076 100644
--- a/src/include/kernel/ipc.H
+++ b/src/include/kernel/ipc.H
@@ -39,7 +39,31 @@ struct ipc_data_area_t
     uint32_t pir;
 };
 
+/**
+ * This stucture maps an IPC data area needed by start_payload that is common
+ * to all HB instances.
+ * @note  Don't change the order or size of this stucture
+ * w/o also changing src/kernel/shutdown.S
+ */
+struct start_payload_data_area_t
+{
+    /** barriers needed in shutdown.S - see src/kernel/shutdown.S
+     *  barrier[0] blocks until all nodes have reported cpu/node_count
+     *  barrier[1] blocks until safe to update HRMOR on master
+     *  barrier[2] blocks until HRMORs updated
+     *  barrier[3] blocks until SLBs, ERATs etc. cleared
+     *  barrier[4] blocks until all CPUs have jumpted to payload except the
+     *             one with the lowest PIR
+     */
+    uint64_t barrier[5];
+    uint64_t lowest_PIR;    //!< Lowest cpu PIR in entire system (all nodes)
+    uint64_t cpu_count;     //!< Total number of cpu threads in entire system
+    uint64_t node_count;    //!< Total number of nodes in entire system
+};
+
+
 extern ipc_data_area_t ipc_data_area;
+extern start_payload_data_area_t start_payload_data_area;
 };
 
 #endif
diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H
index 42cfd3fa0..a7e40356d 100644
--- a/src/include/kernel/misc.H
+++ b/src/include/kernel/misc.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -47,6 +47,8 @@ namespace KernelMisc
     extern uint64_t g_payload_entry;
     /** @brief Address from base of payload data pointer. */
     extern uint64_t g_payload_data;
+    /** @brief master host boot instance number (node) */
+    extern uint64_t g_masterHBInstance;
 
     /** @fn in_kernel_mode
      *  @brief Determine if the code is currently in kernel mode or not.
diff --git a/src/include/sys/misc.h b/src/include/sys/misc.h
index a73398efa..6ccde7882 100644
--- a/src/include/sys/misc.h
+++ b/src/include/sys/misc.h
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -97,11 +97,13 @@ extern "C"
  *                              entry-point.
  *  @param[in] i_payload_data   Data pointer fo the payload.  For standalone
  *                              Saphire this is the devtree
+ *  @param[in[ i_masterHBInstance  Hostboot instance number. for multinode
  */
 extern "C" void shutdown(uint64_t i_status,
                          uint64_t i_payload_base,
                          uint64_t i_payload_entry,
-                         uint64_t i_payload_data);
+                         uint64_t i_payload_data,
+                         uint64_t i_masterHBInstance);
 #endif
 
 /** @enum ProcessorCoreType
diff --git a/src/include/usr/initservice/initserviceif.H b/src/include/usr/initservice/initserviceif.H
index 05231fd59..f32bbc28c 100644
--- a/src/include/usr/initservice/initserviceif.H
+++ b/src/include/usr/initservice/initserviceif.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -30,6 +30,8 @@
 #include    <targeting/namedtarget.H>
 #include    <targeting/attrsync.H>
 
+#define THIS_NODE_NO_PAYLOAD 0xffffffffffffffffull
+
 namespace INITSERVICE
 {
 
@@ -91,6 +93,10 @@ bool unregisterShutdownEvent(msg_q_t i_msgQ);
  *                              payload entry-point.
  * @param[in] i_payload_data - Pointer to payload data (if needed)
  *
+ * @param[in] i_masterHBinstance - master Host boot instance number (node)
+ *                                 Needed when starting payload
+ *                                 on multi-node system.
+ *
  * @return Nothing
  *
  * @note If inBackground = true, the shutdown call will spawn a child task
@@ -102,7 +108,8 @@ void doShutdown ( uint64_t i_status,
                   bool i_inBackground = false,
                   uint64_t i_payload_base = 0,
                   uint64_t i_payload_entry = 0,
-                  uint64_t i_payload_data = 0);
+                  uint64_t i_payload_data = 0,
+                  uint64_t i_masterHBInstance = THIS_NODE_NO_PAYLOAD);
 
 /**
  * @brief Returns if Service Processor Base Services are available
diff --git a/src/include/usr/mbox/ipc_msg_types.H b/src/include/usr/mbox/ipc_msg_types.H
index e6b450b40..3271c515e 100644
--- a/src/include/usr/mbox/ipc_msg_types.H
+++ b/src/include/usr/mbox/ipc_msg_types.H
@@ -34,6 +34,7 @@ namespace IPC
     {
         IPC_POPULATE_ATTRIBUTES = MBOX::FIRST_SECURE_MSG + 1,
         IPC_TEST_CONNECTION = MBOX::FIRST_SECURE_MSG + 2,
+        IPC_START_PAYLOAD,
     };
 
 }; // namespace IPC
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index 738c3fe56..bb5574e2f 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -36,19 +36,23 @@
 #include <kernel/memstate.H>
 #include <kernel/intmsghandler.H>
 #include <kernel/hbdescriptor.H>
+#include <kernel/ipc.H>
 
 extern "C"
     void kernel_shutdown(size_t, uint64_t, uint64_t, uint64_t,
-                         uint64_t) NO_RETURN;
+                         uint64_t, uint64_t) NO_RETURN;
 
 extern HB_Descriptor kernel_hbDescriptor;
 
+KernelIpc::start_payload_data_area_t KernelIpc::start_payload_data_area;
+
 namespace KernelMisc
 {
 
     uint64_t g_payload_base = 0;
     uint64_t g_payload_entry = 0;
     uint64_t g_payload_data = 0;
+    uint64_t g_masterHBInstance = 0xfffffffffffffffful;
 
     void shutdown()
     {
@@ -109,7 +113,7 @@ namespace KernelMisc
             else
             {
                 static Barrier* l_barrier = new Barrier(CpuManager::getCpuCount());
-                static uint64_t l_lowestPIR = 0xffffffffffffffffull;
+                static uint64_t l_lowestPIR = 0xfffffffffffffffful;
 
                 if (c->master)
                 {
@@ -138,20 +142,70 @@ namespace KernelMisc
 
                 l_barrier->wait();
 
+                // only set this to valid PIR if local master
+                // otherwise leave as default;
+                uint64_t local_master_pir = 0xfffffffffffffffful;
+
+                // Find the start_payload_data_area on the master node
+                uint64_t hrmor_base = KernelIpc::ipc_data_area.hrmor_base;
+                uint64_t this_node =
+                    getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
+
+                uint64_t hrmor_offset = getHRMOR() - (this_node * hrmor_base);
+
+                uint64_t dest_hrmor =
+                    (g_masterHBInstance * hrmor_base) + hrmor_offset;
+
+                uint64_t start_payload_data_area_address =
+                    reinterpret_cast<uint64_t>
+                    (&KernelIpc::start_payload_data_area);
+
+                start_payload_data_area_address += dest_hrmor;
+                start_payload_data_area_address |= 0x8000000000000000ul;
+
+                KernelIpc::start_payload_data_area_t * p_spda =
+                    reinterpret_cast<KernelIpc::start_payload_data_area_t*>
+                    (start_payload_data_area_address);
+
                 if (c->master)
                 {
+                    local_master_pir = getPIR();
                     // Reset the memory state register so that the dump tools
                     // don't attempt to dump all of memory once payload runs.
                     KernelMemState::setMemScratchReg(
                             KernelMemState::MEM_CONTAINED_NR,
                             KernelMemState::NO_MEM);
+
+
+                    // add this nodes cpu_count to the system cpu_count
+                    __sync_add_and_fetch(&(p_spda->cpu_count),
+                                         CpuManager::getCpuCount());
+
+                    // set lowest system PIR based on local lowest PIR
+                    do
+                    {
+                        uint64_t currentPIR = p_spda->lowest_PIR;
+                        if (l_lowestPIR > currentPIR)
+                        {
+                            break;
+                        }
+
+                        if (__sync_bool_compare_and_swap(&p_spda->lowest_PIR,
+                                                         currentPIR, l_lowestPIR))
+                        {
+                            break;
+                        }
+
+                    } while(1);
+
                 }
 
-                kernel_shutdown(CpuManager::getCpuCount(),
+                kernel_shutdown(p_spda->node_count,
                                 g_payload_base,
                                 g_payload_entry,
                                 g_payload_data,
-                                l_lowestPIR);
+                                local_master_pir,  //master PIR if local master
+                                start_payload_data_area_address);
             }
         }
         else
diff --git a/src/kernel/shutdown.S b/src/kernel/shutdown.S
index 74492f37d..bee5f0086 100644
--- a/src/kernel/shutdown.S
+++ b/src/kernel/shutdown.S
@@ -5,7 +5,7 @@
 #
 # IBM CONFIDENTIAL
 #
-# COPYRIGHT International Business Machines Corp. 2012,2013
+# COPYRIGHT International Business Machines Corp. 2012,2014
 #
 # p1
 #
@@ -31,6 +31,7 @@
         bne- 1b; \
         isync; \
         /*     Wait for count to reach CPU count. */ \
+    2: \
     1: \
         or 1,1,1;    /* Drop thread priority */ \
         ld temp, 0(addr); \
@@ -51,21 +52,24 @@
     ;// Steps:
     ;//     All threads enter "EA[0]=1" mode.
     ;//     <sync barrier 1>
-    ;//     Thread0 on each core updates HRMOR.
+    ;//     All nodes have reported cpu_count
     ;//     <sync barrier 2>
-    ;//     All threads execute - isync ; slbia ; isync
+    ;//     Thread0 on each core updates HRMOR.
     ;//     <sync barrier 3>
+    ;//     All threads execute - isync ; slbia ; isync
+    ;//     <sync barrier 4>
     ;//     All threads except "last thread" enters payload.
     ;//         * last thread waits for all other threads to leave.
     ;//
     ;// The design of the barrier stages comes from the BookIV chapter
     ;// titled "HRMOR Update Sequence".
     ;//
-    ;// @param[in] r3 - CPU count - Number of active CPUs.
+    ;// @param[in] r3 - Number of Host boot instances (nodes) on the system
     ;// @param[in] r4 - Payload Base
     ;// @param[in] r5 - Payload Entry
     ;// @param[in] r6 - Payload Data
-    ;// @param[in] r7 - Last thread to enter payload.
+    ;// @param[in] r7 - PIR of local master cpu - only set by local master
+    ;// @param[in] r8 - System address of start_payload_data_area
     ;//
 .global kernel_shutdown
 kernel_shutdown:
@@ -74,11 +78,6 @@ kernel_shutdown:
     rotldi r10, r10, 63
         ;// Retrieve existing HRMOR.
     mfspr r0, HRMOR
-        ;// Determine physical address of shutdown_barrier.
-    lis r8, kernel_shutdown_barriers@h
-    ori r8, r8, kernel_shutdown_barriers@l
-    or r8, r8, r0       ;// Apply HRMOR.
-    or r8, r8, r10      ;// Apply EA[0] = 1.
         ;// Determine physical address of EA[0]=1 mode instruction.
     lis r9, kernel_shutdown_ea0_1_mode@h
     ori r9, r9, kernel_shutdown_ea0_1_mode@l
@@ -88,16 +87,33 @@ kernel_shutdown:
     mtlr r9
     blr
 kernel_shutdown_ea0_1_mode:
+
+        ;// Only master cpu on the node should increment barrier for node count
+        ;// as it was the one that updated the cpu_count & lowest_pir
+        ;// barrier - 1 wait for all nodes to report
+    mfspr  r10, PIR
+    cmp    cr0, r10, r7
+    bne+ 2f       ;// inside KERNEL_BARRIER below
+
         ;// Perform barrier - 1
     KERNEL_BARRIER(r8, r3, r11)
 
+    ;// safe now to access system cpu_count and  lowest_PIR
+    ld  r3, 48(r8)   ;// cpu_count for whole system
+    ld  r7, 40(r8)   ;// lowest PIR for whole system
+
+    addi  r8, r8, 8  ;// next barrier
+
+        ;// Perform barrier - 2
+    KERNEL_BARRIER(r8, r3, r11)
+
         ;// Update HRMOR on master.
     mfspr r10, PIR
     andi. r10, r10, 7
     bne+ 1f
     mtspr HRMOR, r4
 1:
-        ;// Perform barrier - 2
+        ;// Perform barrier - 3
     addi r8, r8, 8
     KERNEL_BARRIER(r8, r3, r11)
 
@@ -106,11 +122,11 @@ kernel_shutdown_ea0_1_mode:
     slbia
     isync
 
-        ;// Perform barrier - 3
+        ;// Perform barrier - 4
     addi r8, r8, 8
     KERNEL_BARRIER(r8, r3, r11)
 
-        ;// "Barrier" 4:
+        ;// "Barrier" 5:
         ;//     Increment counter as leaving, except PIR == r7 waits.
     addi r8, r8, 8
             ;// Check for PIR == r7.
@@ -125,6 +141,12 @@ kernel_shutdown_ea0_1_mode:
     bne- 1b
     isync
 2:
+    ;// Clear "Hostboot active" scratch register.
+    li r3, (0x40 + 0x38) ;// See sys/mmio.h
+    mtspr SPRC, r3
+    li r3, 0x0
+    mtspr SPRD, r3
+
     ;// Enter Payload.
         ;// Set HSRR0 to entry point.
     mtspr HSRR0, r5
@@ -156,14 +178,5 @@ kernel_shutdown_ea0_1_mode:
     isync
         ;// Raise thread priority and leave ourselves.
     or 2,2,2
-        ;// Clear "Hostboot active" scratch register.
-    li r3, (0x40 + 0x38) ;// See sys/mmio.h
-    mtspr SPRC, r3
-    li r3, 0x0
-    mtspr SPRD, r3
     b 2b
 
-.section .data
-kernel_shutdown_barriers:
-    .space 8*4 ;//
-
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index ab78bd655..0270644b7 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2010,2013              */
+/* COPYRIGHT International Business Machines Corp. 2010,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -650,6 +650,7 @@ namespace Systemcalls
         KernelMisc::g_payload_base = static_cast<uint64_t>(TASK_GETARG1(t));
         KernelMisc::g_payload_entry = static_cast<uint64_t>(TASK_GETARG2(t));
         KernelMisc::g_payload_data = static_cast<uint64_t>(TASK_GETARG3(t));
+        KernelMisc::g_masterHBInstance= static_cast<uint64_t>(TASK_GETARG4(t));
         CpuManager::requestShutdown(status);
         TASK_SETRTN(t, 0);
     }
diff --git a/src/lib/syscall_misc.C b/src/lib/syscall_misc.C
index c972292f9..d7894d6f8 100644
--- a/src/lib/syscall_misc.C
+++ b/src/lib/syscall_misc.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -31,13 +31,15 @@ using namespace Systemcalls;
 void shutdown(uint64_t i_status,
               uint64_t i_payload_base,
               uint64_t i_payload_entry,
-              uint64_t i_payload_data)
+              uint64_t i_payload_data,
+              uint64_t i_masterHBInstance)
 {
-    _syscall4(MISC_SHUTDOWN,
+    _syscall5(MISC_SHUTDOWN,
                 reinterpret_cast<void*>(i_status),
                 reinterpret_cast<void*>(i_payload_base),
                 reinterpret_cast<void*>(i_payload_entry),
-                reinterpret_cast<void*>(i_payload_data));
+                reinterpret_cast<void*>(i_payload_data),
+                reinterpret_cast<void*>(i_masterHBInstance));
 }
 
 ProcessorCoreType cpu_core_type()
diff --git a/src/usr/hwpf/hwp/start_payload/start_payload.C b/src/usr/hwpf/hwp/start_payload/start_payload.C
index b913ab62c..c64e13b66 100644
--- a/src/usr/hwpf/hwp/start_payload/start_payload.C
+++ b/src/usr/hwpf/hwp/start_payload/start_payload.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -69,6 +69,10 @@
 #include    "start_payload.H"
 #include    <runtime/runtime.H>
 #include    <devtree/devtreeif.H>
+#include    <sys/task.h>
+#include    <kernel/cpu.H> // for KERNEL_MAX_SUPPORTED_CPUS_PER_NODE
+#include    <kernel/ipc.H> // for internode data areas
+#include    <mbox/ipc_msg_types.H>
 
 //  Uncomment these files as they become available:
 // #include    "host_start_payload/host_start_payload.H"
@@ -81,15 +85,27 @@ using   namespace   fapi;
 using   namespace   ISTEP;
 using   namespace   ISTEP_ERROR;
 
+
 /**
  * @brief This function will call the Initservice interface to shutdown
  *      Hostboot.  This function will call shutdown, passing in system
  *      attribute variables for the Payload base and Payload offset.
  *
+ * @param[in] Host boot master instance number (logical node number)
+ * @param[in] Is this the master HB instance [true|false]
+ *
  * @return errlHndl_t - NULL if succesful, otherwise a pointer to the error
  *      log.
  */
-errlHndl_t callShutdown ( void );
+errlHndl_t callShutdown ( uint64_t i_hbInstance, bool i_masterIntance );
+
+/**
+ * @brief This function will send an IPC message to all other HB instances
+ *        to perfrom the shutdown sequence.
+ * @param[in] Hostboot master instance number (logical node number)
+ * @Return errlHndlt_t - Null if succesful, otherwise an error Handle
+ */
+errlHndl_t broadcastShutdown ( uint64_t i_hbInstance );
 
 /**
  * @brief This function will check the Istep mode and send the appropriate
@@ -389,9 +405,17 @@ void*    call_host_start_payload( void    *io_pArgs )
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
             "call_host_start_payload entry" );
 
+    uint64_t this_node =
+        task_getcpuid()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
+
+    // broadcast shutdown to other HB instances.
+    l_errl = broadcastShutdown(this_node);
 
-    //  - Run CXX testcases
-    l_errl = INITSERVICE::executeUnitTests();
+    if( l_errl == NULL)
+    {
+        //  - Run CXX testcases
+        l_errl = INITSERVICE::executeUnitTests();
+    }
 
     if( l_errl == NULL )
     {
@@ -400,7 +424,7 @@ void*    call_host_start_payload( void    *io_pArgs )
         //      - base/entry will be from system attributes
         //      - this will start the payload (Phyp)
         // NOTE: this call will not return if successful.
-        l_errl = callShutdown();
+        l_errl = callShutdown(this_node, true);
 
     };
 
@@ -428,7 +452,8 @@ void*    call_host_start_payload( void    *io_pArgs )
 //
 // Call shutdown
 //
-errlHndl_t callShutdown ( void )
+errlHndl_t callShutdown ( uint64_t i_masterInstance,
+                          bool i_isMaster)
 {
     errlHndl_t err = NULL;
     uint64_t payloadBase = 0x0;
@@ -511,19 +536,23 @@ errlHndl_t callShutdown ( void )
         TARGETING::SpFunctions spFuncs =
                 sys->getAttr<TARGETING::ATTR_SP_FUNCTIONS>();
 
-        // Notify Fsp with appropriate mailbox message.
-        err = notifyFsp( istepModeFlag,
-                         spFuncs );
-
-        if( err )
+        if(i_isMaster)
         {
-            break;
-        }
+            // Notify Fsp with appropriate mailbox message.
+            err = notifyFsp( istepModeFlag,
+                             spFuncs );
 
-        // Load payload data in Sapphire mode when SP Base Services not enabled
-        if( is_sapphire_load() && (!INITSERVICE::spBaseServicesEnabled()) )
-        {
-            payloadData = DEVTREE::get_flatdevtree_phys_addr();
+            if( err )
+            {
+                break;
+            }
+
+            // Load payload data in Sapphire mode when 
+            // SP Base Services not enabled
+            if( is_sapphire_load() && (!INITSERVICE::spBaseServicesEnabled()))
+            {
+                payloadData = DEVTREE::get_flatdevtree_phys_addr();
+            }
         }
 
         // do the shutdown.
@@ -534,7 +563,8 @@ errlHndl_t callShutdown ( void )
                                  false,
                                  payloadBase,
                                  payloadEntry,
-                                 payloadData);
+                                 payloadData,
+                                 i_masterInstance);
 
     } while( 0 );
 
@@ -591,4 +621,104 @@ errlHndl_t notifyFsp ( bool i_istepModeFlag,
     return err;
 }
 
+
+
+
+errlHndl_t broadcastShutdown ( uint64_t i_hbInstance )
+{
+    errlHndl_t err = NULL;
+    TARGETING::Target * sys = NULL;
+    TARGETING::targetService().getTopLevelTarget( sys );
+    assert(sys != NULL);
+
+    TARGETING::ATTR_HB_EXISTING_IMAGE_type hb_images =
+        sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>();
+
+    do
+    {
+        // Set up the start_payload_data_area before
+        // broadcasting the shutdown to the slave HB instances
+        memset(&KernelIpc::start_payload_data_area,
+               '\0',
+               sizeof(KernelIpc::start_payload_data_area));
+
+        KernelIpc::start_payload_data_area.node_count = 0;
+        KernelIpc::start_payload_data_area.lowest_PIR = 0xfffffffffffffffful;
+
+        // ATTR_HB_EXISTING_IMAGE only gets set on a multi-drawer system.
+        // Currently set up in host_sys_fab_iovalid_processing() which only
+        // gets called if there are multiple physical nodes.
+        if(hb_images == 0)
+        {
+            // Single node system
+            break;
+        }
+
+        // continue - multi-node
+
+        uint8_t node_map
+            [sizeof(TARGETING::ATTR_FABRIC_TO_PHYSICAL_NODE_MAP_type)];
+
+        sys->tryGetAttr<TARGETING::ATTR_FABRIC_TO_PHYSICAL_NODE_MAP>(node_map);
+
+        uint64_t node_count = 0;
+
+        // Count the number of hb instances before sending
+        // any start_payload messages
+        for(uint64_t drawer = 0; drawer < sizeof(node_map); ++drawer)
+        {
+            uint64_t node = node_map[drawer];
+
+            if(node < (sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8))
+            {
+
+                // set mask to msb
+                TARGETING::ATTR_HB_EXISTING_IMAGE_type mask = 0x1 <<
+                    ((sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8) -1);
+
+                if( 0 != ((mask >> node) & hb_images ) )
+                {
+                    ++node_count;
+                }
+            } 
+        }
+
+        KernelIpc::start_payload_data_area.node_count = node_count;
+
+        // send message to all other existing hb instances except this one.
+        for(uint64_t drawer = 0; drawer < sizeof(node_map); ++drawer)
+        {
+            uint64_t node = node_map[drawer];
+
+            if(node < (sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8) &&
+               node != i_hbInstance)
+            {
+
+                // set mask to msb
+                TARGETING::ATTR_HB_EXISTING_IMAGE_type mask = 0x1 <<
+                    ((sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8) -1);
+
+                if( 0 != ((mask >> node) & hb_images ) )
+                {
+                    TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                               "start_payload-sending msg for drawer %d",
+                               drawer );
+                    msg_t * msg = msg_allocate();
+                    msg->type = IPC::IPC_START_PAYLOAD;
+                    msg->data[0] = i_hbInstance;
+                    err = MBOX::send(MBOX::HB_IPC_MSGQ, msg, node);
+                    if (err)
+                    {
+                        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "MBOX::send failed");
+                        break;
+                    }
+                }
+            } 
+        }
+
+    } while(0);
+
+    return err;
+}
+
 };   // end namespace
diff --git a/src/usr/hwpf/hwp/start_payload/start_payload.H b/src/usr/hwpf/hwp/start_payload/start_payload.H
index 62727f242..8aed3de40 100644
--- a/src/usr/hwpf/hwp/start_payload/start_payload.H
+++ b/src/usr/hwpf/hwp/start_payload/start_payload.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012                   */
+/* COPYRIGHT International Business Machines Corp. 2012,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -107,8 +107,6 @@ void*    call_host_verify_hdat( void    *io_pArgs );
  */
 void*    call_host_start_payload( void    *io_pArgs );
 
-
-
 };   // end namespace
 
 #endif
diff --git a/src/usr/initservice/baseinitsvc/initservice.C b/src/usr/initservice/baseinitsvc/initservice.C
index 0d34a5abb..ece13d59a 100644
--- a/src/usr/initservice/baseinitsvc/initservice.C
+++ b/src/usr/initservice/baseinitsvc/initservice.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -632,7 +632,8 @@ void doShutdown(uint64_t i_status,
                 bool i_inBackground,
                 uint64_t i_payload_base,
                 uint64_t i_payload_entry,
-                uint64_t i_payload_data)
+                uint64_t i_payload_data,
+                uint64_t i_masterHBInstance)
 {
     class ShutdownExecute
     {
@@ -640,11 +641,13 @@ void doShutdown(uint64_t i_status,
             ShutdownExecute(uint64_t i_status,
                             uint64_t i_payload_base,
                             uint64_t i_payload_entry,
-                            uint64_t i_payload_data)
+                            uint64_t i_payload_data,
+                            uint64_t i_masterHBInstance)
                 : status(i_status),
                   payload_base(i_payload_base),
                   payload_entry(i_payload_entry),
-                  payload_data(i_payload_data)
+                  payload_data(i_payload_data),
+                  masterHBInstance(i_masterHBInstance)
             { }
 
             void execute()
@@ -652,7 +655,8 @@ void doShutdown(uint64_t i_status,
                 Singleton<InitService>::instance().doShutdown(status,
                                                               payload_base,
                                                               payload_entry,
-                                                              payload_data);
+                                                              payload_data,
+                                                              masterHBInstance);
             }
             void startThread()
             {
@@ -664,6 +668,7 @@ void doShutdown(uint64_t i_status,
             uint64_t payload_base;
             uint64_t payload_entry;
             uint64_t payload_data;
+            uint64_t masterHBInstance;
 
             static void* run(void* _self)
             {
@@ -679,7 +684,8 @@ void doShutdown(uint64_t i_status,
     };
 
     ShutdownExecute* s = new ShutdownExecute(i_status, i_payload_base,
-                                             i_payload_entry, i_payload_data);
+                                             i_payload_entry, i_payload_data,
+                                             i_masterHBInstance);
 
     if (i_inBackground)
     {
@@ -695,7 +701,8 @@ void doShutdown(uint64_t i_status,
 void InitService::doShutdown(uint64_t i_status,
                              uint64_t i_payload_base,
                              uint64_t i_payload_entry,
-                             uint64_t i_payload_data)
+                             uint64_t i_payload_data,
+                             uint64_t i_masterHBInstance)
 {
     int l_rc = 0;
     errlHndl_t l_err = NULL;
@@ -757,7 +764,11 @@ void InitService::doShutdown(uint64_t i_status,
         l_rb_iter++;
     }
 
-    shutdown(i_status, i_payload_base, i_payload_entry, i_payload_data);
+    shutdown(i_status,
+             i_payload_base,
+             i_payload_entry,
+             i_payload_data,
+             i_masterHBInstance);
 }
 
 bool InitService::registerShutdownEvent(msg_q_t i_msgQ,
diff --git a/src/usr/initservice/baseinitsvc/initservice.H b/src/usr/initservice/baseinitsvc/initservice.H
index 26e620b95..cf4536a86 100644
--- a/src/usr/initservice/baseinitsvc/initservice.H
+++ b/src/usr/initservice/baseinitsvc/initservice.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -209,6 +209,9 @@ public:
      *                              payload entry-point.
      * @param[in] i_payload_entry - HRMOR adjusted address of any payload data
      *                              placed in r3
+     * @param[in] i_masterHBInstance - master hostboot instance number (node)
+     *                                 Needed when starting payload on a
+     *                                 multi-node system.
      *
      * @return Nothing
      * @note This calls registered services to notify them of shutdown and it
@@ -217,7 +220,8 @@ public:
     void doShutdown ( uint64_t i_status,
                       uint64_t i_payload_base = 0,
                       uint64_t i_payload_entry = 0,
-                      uint64_t i_payload_data = 0);
+                      uint64_t i_payload_data = 0,
+                      uint64_t i_masterHBInstance = 0xffffffffffffffffull);
 
 protected:
 
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index e0b2a3449..ac45be232 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -29,6 +29,12 @@
 #include <errl/errlmanager.H>
 #include <mbox/mbox_reasoncodes.H>
 
+namespace START_PAYLOAD
+{
+    extern errlHndl_t callShutdown ( uint64_t i_hbInstance,
+                                     bool i_masterInstance );
+};
+
 trace_desc_t* g_trac_ipc = NULL;
 TRAC_INIT(&g_trac_ipc, IPC_TRACE_NAME, KILOBYTE);
 
@@ -118,6 +124,7 @@ void IpcSp::msgHandler()
                     {
                         errlCommit(err, IPC_COMP_ID);
                     }
+                    mod_loaded = false;
                 }
 
                 // Respond
@@ -143,6 +150,51 @@ void IpcSp::msgHandler()
                     errlCommit(err,IPC_COMP_ID);
                 }
                 break;
+
+            case IPC_START_PAYLOAD:
+
+                if ( !VFS::module_is_loaded( "libstart_payload.so" ) )
+                {
+                    err = VFS::module_load( "libstart_payload.so" );
+
+                    if ( err )
+                    {
+                        TRACFCOMP( g_trac_ipc,
+                                   "Could not load runtime module" );
+                    }
+                    else
+                    {
+                        mod_loaded = true;
+                    }
+                }
+
+                if(!err)
+                {
+                    //  Function will not return unless error
+                    err = START_PAYLOAD::callShutdown(msg->data[0],false);
+                }
+
+                if(err)
+                {
+                    errlCommit(err, IPC_COMP_ID);
+                }
+
+                if(mod_loaded)
+                {
+                    err = VFS::module_unload( "libstart_payload.so" );
+
+                    if (err)
+                    {
+                        errlCommit(err, IPC_COMP_ID);
+                    }
+                    mod_loaded = false;
+                }
+
+                msg_free(msg);
+
+                break;
+
+
             default:
 
                 TRACFCOMP( g_trac_ipc,
@@ -181,6 +233,8 @@ void IpcSp::msgHandler()
 
                 errlCommit(err, IPC_COMP_ID);
 
+                msg_free(msg);
+
                 break;
         }
     }