Add retry to slave core wakeup path

We are still seeing some very intermittent errors in the slave
core wakeup path.  It still seems like we may have a timing issue.
Until we figure out exactly what is going on, I am adding a retry
mechanism that should get the core to report in correctly.  The
retry is done by issuing an additional doorbell message to the
core that didn't report in.

Change-Id: Ib87e5d58e079674d1eebb44c10d0252a35ea0519
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/70761
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Dean Sanner <dsanner@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 46 insertions, 2 deletions

diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index a2dff9415..425cc2d28 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -454,7 +454,7 @@ void CpuManager::startCore(uint64_t pir,uint64_t i_threads)
         // Only wakeup the threads we were told to wakeup
         if( i_threads & (0x8000000000000000 >> i) )
         {
-            printk("Dbell pir 0x%lx\n", pir + i);
+            printk("Dbell:0x%lx\n", pir + i);
             //Initiate the Doorbell for this core/pir
             send_doorbell_wakeup(pir + i);
         }
@@ -463,6 +463,39 @@ void CpuManager::startCore(uint64_t pir,uint64_t i_threads)
     return;
 };
 
+void CpuManager::wakeupCore(uint64_t pir,uint64_t i_threads)
+{
+    size_t threads = getThreadCount();
+    pir = pir & ~(threads-1);
+
+    if (pir >=
+        (KERNEL_MAX_SUPPORTED_NODES * KERNEL_MAX_SUPPORTED_CPUS_PER_NODE))
+    {
+        TASK_SETRTN(TaskManager::getCurrentTask(), -ENXIO);
+        return;
+    }
+
+    //Send a message to userspace that a core with this base pir is being added
+    // userspace will know which threads on the core to expect already
+    InterruptMsgHdlr::addCpuCore(pir);
+
+    // Physically wakeup the threads with doorbells
+    //  Assumption is that startCore has already run so all
+    //  internal structures are setup
+    for(size_t i = 0; i < threads; i++)
+    {
+        // Only wakeup the threads we were told to wakeup
+        if( i_threads & (0x8000000000000000 >> i) )
+        {
+            printk("Dbell2:0x%lx\n", pir + i);
+            //Initiate the Doorbell for this core/pir
+            doorbell_send(pir + i);
+        }
+    }
+
+    return;
+};
+
 size_t CpuManager::getThreadCount()
 {
     size_t threads = 0;
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index c293d5067..1df43b78e 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -52,6 +52,8 @@
 extern "C"
 void kernel_execute_hyp_doorbell()
 {
+    printkd("hyp_doorbell on %lx\n", getPIR());
+
     // Per POWER ISA Section 5.9.2, to avoid any weak consistency
     //  issues we must use a msgsync instruction before consuming
     //  any data set by a different thread following a doorbell
@@ -144,6 +146,7 @@ namespace Systemcalls
     void CpuSprSet(task_t *t);
     void CpuNap(task_t *t);
     void CpuWinkle(task_t *t);
+    void CpuWakeupCore(task_t *t);
     void MmAllocBlock(task_t *t);
     void MmRemovePages(task_t *t);
     void MmSetPermission(task_t *t);
@@ -189,6 +192,7 @@ namespace Systemcalls
         &CpuSprSet,   // MISC_CPUSPRSET
         &CpuNap, // MISC_CPUNAP
         &CpuWinkle,   // MISC_CPUWINKLE
+        &CpuWakeupCore,   // MISC_CPUWAKEUPCORE
 
         &MmAllocBlock, // MM_ALLOC_BLOCK
         &MmRemovePages, // MM_REMOVE_PAGES
@@ -858,6 +862,13 @@ namespace Systemcalls
         }
     }
 
+    /** Force thread wakeup via doorbell. */
+    void CpuWakeupCore(task_t *t)
+    {
+        CpuManager::wakeupCore(static_cast<uint64_t>(TASK_GETARG0(t)),
+                               static_cast<uint64_t>(TASK_GETARG1(t)));
+    };
+
     /**
      * Allocate a block of virtual memory within the base segment
      * @param[in] t: The task used to allocate a block in the base segment