Updates to make multinode IPC work

 -Fixed node/group id calculations
 -Reduced dbell printk to prevent printk overflow
 -Fixed architectual hole in how internode IPC works
   Workitems won't work, instead just always check
   for IPC on any doorbells to master thread
 -Changed PIR tracing to print out in hex

Change-Id: I25eb7f87fd812a90f98a7724b1ac1100f764fe7b
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/53187
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Dean Sanner <dsanner@us.ibm.com>

10 files changed, 42 insertions, 44 deletions

diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index b967c1319..f218e3e45 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -38,8 +38,11 @@
 #include <util/lockfree/stack.H>
 
 // See BookIV PIR definition.
-#define KERNEL_MAX_SUPPORTED_NODES 8
-#define KERNEL_MAX_SUPPORTED_CPUS_PER_NODE (8 * 16 * 8) // Chip, core, thread.
+#define KERNEL_MAX_SUPPORTED_NODES 4
+
+// This constant is coming from the PIR: GGGGPPPRCCCCCTT
+// 3-bits for chiP (P), 1-reserved bit, 5-Core bits (C), 2-for Thread (T)
+#define KERNEL_MAX_SUPPORTED_CPUS_PER_NODE (8 * 2 * 32 * 4) // ChipId, reserved bit, core, thread
 
 class Scheduler;
 
diff --git a/src/include/kernel/workitem.H b/src/include/kernel/workitem.H
index 6d2f4b6b1..856f5b365 100644
--- a/src/include/kernel/workitem.H
+++ b/src/include/kernel/workitem.H
@@ -54,16 +54,5 @@ class CpuWakeupDoorbellWorkItem : public KernelWorkItem
 };
 
 
-
-//A work item to be created/executed during an IPC scenario
-class IPCDoorbellWorkItem : public KernelWorkItem
-{
-    public:
-        //Implement operator() function
-        void operator() (void);
-
-        //No data to clean up, use default destructor
-        ~IPCDoorbellWorkItem() = default;
-};
 #endif
 
diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index 2469ad36b..095b0a946 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -195,7 +195,7 @@ void CpuManager::startCPU(ssize_t i)
     // Initialize CPU structure.
     if (NULL == cv_cpus[nodeId][cpuId])
     {
-        printk("Starting CPU with pir %ld...", i);
+        printkd("Start pir 0x%lx...", i);
         cpu_t* cpu = cv_cpus[nodeId][cpuId] = new cpu_t();
 
         // Initialize CPU.
@@ -257,7 +257,7 @@ void CpuManager::startCPU(ssize_t i)
         // Call TimeManager setup for a CPU.
         TimeManager::init_cpu(cpu);
 
-        printk("done\n");
+        printkd("done\n");
     }
 
     if (currentCPU)
@@ -446,7 +446,7 @@ void CpuManager::startCore(uint64_t pir,uint64_t i_threads)
         // Only wakeup the threads we were told to wakeup
         if( i_threads & (0x8000000000000000 >> i) )
         {
-            printk("Dbell wkup pir %ld\n", pir + i);
+            printk("Dbell pir 0x%lx\n", pir + i);
             //Initiate the Doorbell for this core/pir
             send_doorbell_wakeup(pir + i);
         }
diff --git a/src/kernel/doorbell.C b/src/kernel/doorbell.C
index 010b7e078..e746ab8cb 100644
--- a/src/kernel/doorbell.C
+++ b/src/kernel/doorbell.C
@@ -81,12 +81,6 @@ void send_doorbell_wakeup(uint64_t i_pir)
 
 void send_doorbell_ipc(uint64_t i_pir)
 {
-    cpu_t *l_cpu = CpuManager::getCpu(i_pir);
-
-    printkd("send_doorbell_ipc to pir: %lx\n", i_pir);
-    //Create WorkItem and put on the stack to be executed during doorbell
-    //   execution
-    KernelWorkItem* l_work = new IPCDoorbellWorkItem();
-    l_cpu->doorbell_actions.push(l_work);
+    printk("send_doorbell_ipc to pir: %lx\n", i_pir);
     doorbell_send(i_pir);
 }
diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C
index f7df4aacb..f097f83da 100644
--- a/src/kernel/ipc.C
+++ b/src/kernel/ipc.C
@@ -86,7 +86,7 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     p_dest->msg_queue_id = i_q;    // set destination queue id
     lwsync();
 
-    printkd("IPC send from PIR %lx to PIR %x\n",getPIR(),p_dest->pir);
+    printk("IPC to PIR %x\n",p_dest->pir);
 
     // send doorbell to interrupt the other drawer
     send_doorbell_ipc(p_dest->pir);
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index d1ff0c45a..3ab2a7a57 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -154,8 +154,9 @@ namespace KernelMisc
 
                 // Find the start_payload_data_area on the master node
                 uint64_t hrmor_base = KernelIpc::ipc_data_area.hrmor_base;
+
                 uint64_t this_hb_instance =
-                    l_lowestPIR/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
+                  l_lowestPIR/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
 
                 uint64_t hrmor_offset =
                     getHRMOR() - (this_hb_instance * hrmor_base);
@@ -177,6 +178,9 @@ namespace KernelMisc
                 if (c->master)
                 {
                     local_master_pir = getPIR();
+
+                    printkd("Local master pir %lx, start_data_area %lx\n",
+                           local_master_pir, start_payload_data_area_address);
                     // Reset the memory state register so that the dump tools
                     // don't attempt to dump all of memory once payload runs.
                     KernelMemState::setMemScratchReg(
diff --git a/src/kernel/start.S b/src/kernel/start.S
index 9b61720a3..40ff3b0ed 100644
--- a/src/kernel/start.S
+++ b/src/kernel/start.S
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2010,2017
+# Contributors Listed Below - COPYRIGHT 2010,2018
 # [+] International Business Machines Corp.
 # [+] Joel Stanley
 #
@@ -319,7 +319,7 @@ _other_thread_spinlock_complete:
     lis r2, _ZN10CpuManager7cv_cpusE@h
     ori r2, r2, _ZN10CpuManager7cv_cpusE@l
     mfspr r1, PIR               ;// Extract node id.
-    extrwi r1, r1, 3, 19
+    extrwi r1, r1, 3, 18        ;// Match PIR format(KERNEL_MAX_SUPPORTED_CPUS_PER_NODE)
     sldi r1, r1, 3
     ldx r2, r1, r2              ;// Dereference to get on-node CPUs array.
     cmpwi cr0, r2, 0            ;// Check for NULL node array.
@@ -610,7 +610,7 @@ intvect_system_reset:
     lis r2, _ZN10CpuManager7cv_cpusE@h
     ori r2, r2, _ZN10CpuManager7cv_cpusE@l
     mfspr r1, PIR               ;// Extract node id.
-    extrwi r1, r1, 3, 19
+    extrwi r1, r1, 3, 18        ;// Match PIR format(KERNEL_MAX_SUPPORTED_CPUS_PER_NODE)
     sldi r1, r1, 3
     ldx r2, r1, r2              ;// Dereference to get on-node CPUs array.
     cmpwi cr0, r2, 0            ;// Check for NULL node array.
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index d4df30733..04c65eb10 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -67,6 +67,18 @@ void kernel_execute_hype_doorbell()
         l_work = t->cpu->doorbell_actions.pop();
     }
 
+    //IPC messages come in only on the master, so
+    //If this is a doorbell to the master -- check
+    cpu_t* master = CpuManager::getMasterCPU();
+    if(t->cpu == master)
+    {
+        size_t pir = getPIR();
+        printk("IPC msg pir %lx incoming\n", pir);
+        //Send message to the intrrp in userspace indicating it has
+        // potential a pending IPC message.
+        InterruptMsgHdlr::sendIpcMsg(pir);
+    }
+
     if (t->cpu->idle_task == t)
     {
         t->cpu->scheduler->returnRunnable();
diff --git a/src/kernel/workitem.C b/src/kernel/workitem.C
index a66908514..0edf83083 100644
--- a/src/kernel/workitem.C
+++ b/src/kernel/workitem.C
@@ -33,20 +33,10 @@
 void CpuWakeupDoorbellWorkItem::operator() (void)
 {
     size_t pir = getPIR();
-    printk("Wkup pir %ld done\n", pir);
+    printkd("Wkup pir %ld done\n", pir);
     //Send message to the intrrp in userspace indicating this pir has woken up
     // There is a task associated with the intrrp that monitors that the proper
     // cores/threads have woken up
     InterruptMsgHdlr::sendThreadWakeupMsg(pir);
     return;
 }
-
-void IPCDoorbellWorkItem::operator() (void)
-{
-    size_t pir = getPIR();
-    printk("IPC msg pir %ld done\n", pir);
-    //Send message tot he intrrp in userspace indicating it has a pending IPC
-    // message.
-    InterruptMsgHdlr::sendIpcMsg(pir);
-    return;
-}
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index a076e0fb5..d5f00f8ef 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -937,7 +937,7 @@ void IntrRp::msgHandler()
 
                     TRACFCOMP(g_trac_intr,INFO_MRK
                               "IntrRp::msgHandler Doorbell IPC msg received"
-                                  " for %d", l_xirr_pir);
+                                  " for %x", l_xirr_pir);
 
                     // Now handle any IPC messages
                     // If something is registered for the IPC msg
@@ -1000,6 +1000,12 @@ void IntrRp::msgHandler()
                            "to handle it. Ignoring it.");
                     }
 
+                    // Always acknowlege msg to kernel
+                    // kernel expects rc in data[1]
+                    // rc of 0 means a successful return
+                    msg->data[1] = 0;
+                    msg_respond(iv_msgQ, msg);
+
                     if (l_err)
                     {
                         l_err->collectTrace(INTR_COMP_NAME, 256);
@@ -1212,18 +1218,18 @@ void IntrRp::msgHandler()
 
                 }
                 break;
-#ifdef CONFIG_MPIPL_ENABLED //TODO RTC 134431
             case MSG_INTR_ADD_HBNODE:  // node info for mpipl
                 {
+#ifdef CONFIG_MPIPL_ENABLED //TODO RTC 134431
                     errlHndl_t err = addHbNodeToMpiplSyncArea(msg->data[0]);
                     if(err)
                     {
                         errlCommit(err,INTR_COMP_ID);
                     }
+#endif
                     msg_free(msg); // async message
                 }
                 break;
-#endif
             case MSG_INTR_DRAIN_QUEUE:
                 {
                     //The purpose of this message is allow the