Save HRMOR in mbox scratch reg for IPC messaging

In order to know where the IPC message of a given node is,
we save off the HRMOR of every node in a hw register.
Originally, we were saving this information in the core
scratch register. Since, the core scratch registers are
wiped off when the cores go into the winkle state, therefore,
we were writing to the register after we come out of winkle.
But, at that point, we ran into race conditions because
other nodes could be ahead and try to access the register
on a node that is not exactly out of winkle yet.

This fixes the problems by using the mbox scratch register
rather than the core scratch register because they are saved
off even when the cores go into winkle state. Because the
registers are preseved, we can set the value prior to
cores coming out of winkle, so, we don't run into the race
condition where one node is trying to read a value prior to
the other one writing the value.

Change-Id: I822bfc8defe09cbb418edc5f36a99b7cd41eec88
CQ:SW435271
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/61093
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

1 files changed, 22 insertions, 35 deletions

diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index 2436c1f52..aa6b626aa 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -32,12 +32,13 @@
 #include <mbox/mbox_reasoncodes.H>
 #include <intr/interrupt.H>
 #include <initservice/initserviceif.H>
+#include <initservice/mboxRegs.H>
 #include <sbeio/sbeioif.H>
 #include <util/utiltce.H>
+#include <util/utilmbox_scratch.H>
 #include <targeting/targplatutil.H>
 #include <targeting/common/targetservice.H>
 #include <targeting/common/attributes.H>
-#include <p9_quad_scom_addresses.H>
 #include <sys/internode.h>
 #include <sys/mmio.h>
 #include <xscom/xscomif.H>
@@ -52,7 +53,7 @@ namespace ISTEP_21
 };
 
 trace_desc_t* g_trac_ipc = NULL;
-TRAC_INIT(&g_trac_ipc, IPC_TRACE_NAME, KILOBYTE);
+TRAC_INIT(&g_trac_ipc, IPC_TRACE_NAME, 4*KILOBYTE);
 
 using namespace IPC;
 using namespace ERRORLOG;
@@ -77,41 +78,16 @@ void IpcSp::init(errlHndl_t & o_errl)
 
 void IpcSp::distributeLocalNodeAddr( void )
 {
-    // Store IPC address for local node in core scratch registers
+    // Store IPC address for local node in mbox scratch register 7
     //  to identify IPC msg address to remote node(s)
     uint64_t l_localNode;
     uint64_t l_remoteAddr;
     qryLocalIpcInfo( l_localNode, l_remoteAddr );
 
-    TARGETING::Target * l_pSys = NULL;
-    TARGETING::targetService().getTopLevelTarget( l_pSys );
-    TARGETING::TargetHandleList l_coreTargetList;
-    TARGETING::getChildChiplets( l_coreTargetList,
-                                 l_pSys,
-                                 TARGETING::TYPE_CORE,
-                                 true );
-
-    // Store IPC address into scom reg for each core
-    // Every core on this node needs to have the value stored
-    //  in it's scratch register in case any cores get deconfigured
-    for(const auto & l_core_target : l_coreTargetList)
-    {
-        uint64_t l_remoteAddrSize = sizeof(l_remoteAddr);
-        errlHndl_t l_err = deviceWrite( l_core_target,
-                                        &l_remoteAddr,
-                                        l_remoteAddrSize,
-                                        DEVICE_SCOM_ADDRESS(C_SCR2) );
-
-        if (l_err)
-        {
-            TRACFCOMP( g_trac_ipc,
-                   "ERROR: distributeLocalNodeAddr == failed to scom Addr=0x%x"
-                   " Target=0x%x", C_SCR2, get_huid(l_core_target));
-            errlCommit(l_err, IPC_COMP_ID);
-        }
-    }
-
-    return;
+    Util::writeScratchReg (INITSERVICE::SPLESS::MBOX_SCRATCH_REG7,
+                           l_remoteAddr>>32);
+    Util::writeScratchReg (INITSERVICE::SPLESS::MBOX_SCRATCH_REG8,
+                           l_remoteAddr);
 }
 
 void IpcSp::acquireRemoteNodeAddrs( void )
@@ -681,9 +657,20 @@ void IpcSp::_acquireRemoteNodeAddrs( void )
                           ( (validNodeBitMap & (0x80 >> i)) != 0  )
                         {
                             // read scoms for remote node
-                            l_RemoteAddr =
-                                    XSCOM::readRemoteCoreScomMultiCast(i,
-                                                                       C_SCR2);
+                            uint64_t l_remoteAddrHighBits =
+                                    XSCOM::readRemoteScom(i,
+                                    INITSERVICE::SPLESS::MBOX_SCRATCH_REG7);
+
+                            uint64_t l_remoteAddrLowBits =
+                                    XSCOM::readRemoteScom(i,
+                                    INITSERVICE::SPLESS::MBOX_SCRATCH_REG8);
+
+                            l_RemoteAddr = (l_remoteAddrHighBits ) |
+                                           (l_remoteAddrLowBits >> 32);
+
+                            TRACFCOMP( g_trac_ipc,"readRemoteScom"
+                                    " node=%d, remoteAddr=0x%x",
+                                    i, l_RemoteAddr);
                         } // end valid node
                         else
                         {