implement start_payload for multi-drawer

RTC: 71994
Change-Id: I422f349d5588731a5e7cfc504d96e497958d6b95
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/7426
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

3 files changed, 96 insertions, 28 deletions

diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index 738c3fe56..bb5574e2f 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -36,19 +36,23 @@
 #include <kernel/memstate.H>
 #include <kernel/intmsghandler.H>
 #include <kernel/hbdescriptor.H>
+#include <kernel/ipc.H>
 
 extern "C"
     void kernel_shutdown(size_t, uint64_t, uint64_t, uint64_t,
-                         uint64_t) NO_RETURN;
+                         uint64_t, uint64_t) NO_RETURN;
 
 extern HB_Descriptor kernel_hbDescriptor;
 
+KernelIpc::start_payload_data_area_t KernelIpc::start_payload_data_area;
+
 namespace KernelMisc
 {
 
     uint64_t g_payload_base = 0;
     uint64_t g_payload_entry = 0;
     uint64_t g_payload_data = 0;
+    uint64_t g_masterHBInstance = 0xfffffffffffffffful;
 
     void shutdown()
     {
@@ -109,7 +113,7 @@ namespace KernelMisc
             else
             {
                 static Barrier* l_barrier = new Barrier(CpuManager::getCpuCount());
-                static uint64_t l_lowestPIR = 0xffffffffffffffffull;
+                static uint64_t l_lowestPIR = 0xfffffffffffffffful;
 
                 if (c->master)
                 {
@@ -138,20 +142,70 @@ namespace KernelMisc
 
                 l_barrier->wait();
 
+                // only set this to valid PIR if local master
+                // otherwise leave as default;
+                uint64_t local_master_pir = 0xfffffffffffffffful;
+
+                // Find the start_payload_data_area on the master node
+                uint64_t hrmor_base = KernelIpc::ipc_data_area.hrmor_base;
+                uint64_t this_node =
+                    getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
+
+                uint64_t hrmor_offset = getHRMOR() - (this_node * hrmor_base);
+
+                uint64_t dest_hrmor =
+                    (g_masterHBInstance * hrmor_base) + hrmor_offset;
+
+                uint64_t start_payload_data_area_address =
+                    reinterpret_cast<uint64_t>
+                    (&KernelIpc::start_payload_data_area);
+
+                start_payload_data_area_address += dest_hrmor;
+                start_payload_data_area_address |= 0x8000000000000000ul;
+
+                KernelIpc::start_payload_data_area_t * p_spda =
+                    reinterpret_cast<KernelIpc::start_payload_data_area_t*>
+                    (start_payload_data_area_address);
+
                 if (c->master)
                 {
+                    local_master_pir = getPIR();
                     // Reset the memory state register so that the dump tools
                     // don't attempt to dump all of memory once payload runs.
                     KernelMemState::setMemScratchReg(
                             KernelMemState::MEM_CONTAINED_NR,
                             KernelMemState::NO_MEM);
+
+
+                    // add this nodes cpu_count to the system cpu_count
+                    __sync_add_and_fetch(&(p_spda->cpu_count),
+                                         CpuManager::getCpuCount());
+
+                    // set lowest system PIR based on local lowest PIR
+                    do
+                    {
+                        uint64_t currentPIR = p_spda->lowest_PIR;
+                        if (l_lowestPIR > currentPIR)
+                        {
+                            break;
+                        }
+
+                        if (__sync_bool_compare_and_swap(&p_spda->lowest_PIR,
+                                                         currentPIR, l_lowestPIR))
+                        {
+                            break;
+                        }
+
+                    } while(1);
+
                 }
 
-                kernel_shutdown(CpuManager::getCpuCount(),
+                kernel_shutdown(p_spda->node_count,
                                 g_payload_base,
                                 g_payload_entry,
                                 g_payload_data,
-                                l_lowestPIR);
+                                local_master_pir,  //master PIR if local master
+                                start_payload_data_area_address);
             }
         }
         else
diff --git a/src/kernel/shutdown.S b/src/kernel/shutdown.S
index 74492f37d..bee5f0086 100644
--- a/src/kernel/shutdown.S
+++ b/src/kernel/shutdown.S
@@ -5,7 +5,7 @@
 #
 # IBM CONFIDENTIAL
 #
-# COPYRIGHT International Business Machines Corp. 2012,2013
+# COPYRIGHT International Business Machines Corp. 2012,2014
 #
 # p1
 #
@@ -31,6 +31,7 @@
         bne- 1b; \
         isync; \
         /*     Wait for count to reach CPU count. */ \
+    2: \
     1: \
         or 1,1,1;    /* Drop thread priority */ \
         ld temp, 0(addr); \
@@ -51,21 +52,24 @@
     ;// Steps:
     ;//     All threads enter "EA[0]=1" mode.
     ;//     <sync barrier 1>
-    ;//     Thread0 on each core updates HRMOR.
+    ;//     All nodes have reported cpu_count
     ;//     <sync barrier 2>
-    ;//     All threads execute - isync ; slbia ; isync
+    ;//     Thread0 on each core updates HRMOR.
     ;//     <sync barrier 3>
+    ;//     All threads execute - isync ; slbia ; isync
+    ;//     <sync barrier 4>
     ;//     All threads except "last thread" enters payload.
     ;//         * last thread waits for all other threads to leave.
     ;//
     ;// The design of the barrier stages comes from the BookIV chapter
     ;// titled "HRMOR Update Sequence".
     ;//
-    ;// @param[in] r3 - CPU count - Number of active CPUs.
+    ;// @param[in] r3 - Number of Host boot instances (nodes) on the system
     ;// @param[in] r4 - Payload Base
     ;// @param[in] r5 - Payload Entry
     ;// @param[in] r6 - Payload Data
-    ;// @param[in] r7 - Last thread to enter payload.
+    ;// @param[in] r7 - PIR of local master cpu - only set by local master
+    ;// @param[in] r8 - System address of start_payload_data_area
     ;//
 .global kernel_shutdown
 kernel_shutdown:
@@ -74,11 +78,6 @@ kernel_shutdown:
     rotldi r10, r10, 63
         ;// Retrieve existing HRMOR.
     mfspr r0, HRMOR
-        ;// Determine physical address of shutdown_barrier.
-    lis r8, kernel_shutdown_barriers@h
-    ori r8, r8, kernel_shutdown_barriers@l
-    or r8, r8, r0       ;// Apply HRMOR.
-    or r8, r8, r10      ;// Apply EA[0] = 1.
         ;// Determine physical address of EA[0]=1 mode instruction.
     lis r9, kernel_shutdown_ea0_1_mode@h
     ori r9, r9, kernel_shutdown_ea0_1_mode@l
@@ -88,16 +87,33 @@ kernel_shutdown:
     mtlr r9
     blr
 kernel_shutdown_ea0_1_mode:
+
+        ;// Only master cpu on the node should increment barrier for node count
+        ;// as it was the one that updated the cpu_count & lowest_pir
+        ;// barrier - 1 wait for all nodes to report
+    mfspr  r10, PIR
+    cmp    cr0, r10, r7
+    bne+ 2f       ;// inside KERNEL_BARRIER below
+
         ;// Perform barrier - 1
     KERNEL_BARRIER(r8, r3, r11)
 
+    ;// safe now to access system cpu_count and  lowest_PIR
+    ld  r3, 48(r8)   ;// cpu_count for whole system
+    ld  r7, 40(r8)   ;// lowest PIR for whole system
+
+    addi  r8, r8, 8  ;// next barrier
+
+        ;// Perform barrier - 2
+    KERNEL_BARRIER(r8, r3, r11)
+
         ;// Update HRMOR on master.
     mfspr r10, PIR
     andi. r10, r10, 7
     bne+ 1f
     mtspr HRMOR, r4
 1:
-        ;// Perform barrier - 2
+        ;// Perform barrier - 3
     addi r8, r8, 8
     KERNEL_BARRIER(r8, r3, r11)
 
@@ -106,11 +122,11 @@ kernel_shutdown_ea0_1_mode:
     slbia
     isync
 
-        ;// Perform barrier - 3
+        ;// Perform barrier - 4
     addi r8, r8, 8
     KERNEL_BARRIER(r8, r3, r11)
 
-        ;// "Barrier" 4:
+        ;// "Barrier" 5:
         ;//     Increment counter as leaving, except PIR == r7 waits.
     addi r8, r8, 8
             ;// Check for PIR == r7.
@@ -125,6 +141,12 @@ kernel_shutdown_ea0_1_mode:
     bne- 1b
     isync
 2:
+    ;// Clear "Hostboot active" scratch register.
+    li r3, (0x40 + 0x38) ;// See sys/mmio.h
+    mtspr SPRC, r3
+    li r3, 0x0
+    mtspr SPRD, r3
+
     ;// Enter Payload.
         ;// Set HSRR0 to entry point.
     mtspr HSRR0, r5
@@ -156,14 +178,5 @@ kernel_shutdown_ea0_1_mode:
     isync
         ;// Raise thread priority and leave ourselves.
     or 2,2,2
-        ;// Clear "Hostboot active" scratch register.
-    li r3, (0x40 + 0x38) ;// See sys/mmio.h
-    mtspr SPRC, r3
-    li r3, 0x0
-    mtspr SPRD, r3
     b 2b
 
-.section .data
-kernel_shutdown_barriers:
-    .space 8*4 ;//
-
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index ab78bd655..0270644b7 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2010,2013              */
+/* COPYRIGHT International Business Machines Corp. 2010,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -650,6 +650,7 @@ namespace Systemcalls
         KernelMisc::g_payload_base = static_cast<uint64_t>(TASK_GETARG1(t));
         KernelMisc::g_payload_entry = static_cast<uint64_t>(TASK_GETARG2(t));
         KernelMisc::g_payload_data = static_cast<uint64_t>(TASK_GETARG3(t));
+        KernelMisc::g_masterHBInstance= static_cast<uint64_t>(TASK_GETARG4(t));
         CpuManager::requestShutdown(status);
         TASK_SETRTN(t, 0);
     }