Save HRMOR in mbox scratch reg for IPC messaging

In order to know where the IPC message of a given node is,
we save off the HRMOR of every node in a hw register.
Originally, we were saving this information in the core
scratch register. Since, the core scratch registers are
wiped off when the cores go into the winkle state, therefore,
we were writing to the register after we come out of winkle.
But, at that point, we ran into race conditions because
other nodes could be ahead and try to access the register
on a node that is not exactly out of winkle yet.

This fixes the problems by using the mbox scratch register
rather than the core scratch register because they are saved
off even when the cores go into winkle state. Because the
registers are preseved, we can set the value prior to
cores coming out of winkle, so, we don't run into the race
condition where one node is trying to read a value prior to
the other one writing the value.

Change-Id: I822bfc8defe09cbb418edc5f36a99b7cd41eec88
CQ:SW435271
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/61093
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

1 files changed, 12 insertions, 22 deletions

diff --git a/src/usr/xscom/xscom.C b/src/usr/xscom/xscom.C
index 7c8bd607c..73d7e9e22 100644
--- a/src/usr/xscom/xscom.C
+++ b/src/usr/xscom/xscom.C
@@ -350,8 +350,7 @@ errlHndl_t  xScomDoOp(DeviceFW::OperationType i_opType,
     uint64_t l_data = 0;
 
     // retry counter.
-    uint32_t l_retryCtr = 0;
-    uint32_t l_retryTraceCtr = 128;
+    uint32_t l_retryCtr = 1;
 
     errlHndl_t l_err = NULL;
 
@@ -378,10 +377,9 @@ errlHndl_t  xScomDoOp(DeviceFW::OperationType i_opType,
         // Check for error or done
         io_hmer = waitForHMERStatus();
 
-        l_retryCtr++;
 
         // If the retry counter is a multiple of 128,256,512,etc.
-        if (l_retryCtr % l_retryTraceCtr*2 == 0)
+        if (l_retryCtr % 100000 == 0)
         {
             // print a trace message.. for debug purposes
             // incase we are stuck in a retry loop.
@@ -395,6 +393,7 @@ errlHndl_t  xScomDoOp(DeviceFW::OperationType i_opType,
                 break;
             }
         }
+        l_retryCtr++;
     } while (io_hmer.mXSComStatus == PIB::PIB_RESOURCE_OCCUPIED);
 
 
@@ -822,10 +821,10 @@ uint64_t generate_mmio_addr( TARGETING::Target* i_proc,
 
 
 /**
- * @brief Multicast Read of core XSCOM register on remote Node
+ * @brief Read of XSCOM register on remote Node
  */
-uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
-                                      uint64_t i_scomAddr )
+uint64_t readRemoteScom( uint64_t i_node,
+                         uint64_t i_scomAddr )
 {
     // definitions of 64 bit xscom address contents that are
     //   useful for this function
@@ -853,10 +852,6 @@ uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
     //    - rsvd                               38
     //  multicast group           0x0000_0000__0700_0000
     //  relative scomAddr field   0x0000_0000__00FF_FFFF
-    constexpr uint64_t XSCOM_MULTICAST =                  0x0000000040000000;
-    constexpr uint64_t XSCOM_MULTICAST_OP_READ_OR =       0x0000000000000000;
-    constexpr uint64_t XSCOM_MULTICAST_GROUP_CORE =       0x0000000001000000;
-    constexpr uint64_t XSCOM_MULTICAST_REL_ADDR_MASK =    0x0000000000FFFFFF;
 
     // Symmetry between nodes is enforced so we know the remote
     //  node contains this chip
@@ -866,17 +861,15 @@ uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
     uint8_t l_chipId =
             l_MasterProcTarget->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
 
-    // compute xscom address & control, then map into processor space
+    // compute xscom address & control
+    // This will return xscom base of the remote node
     uint64_t l_xscomBaseAddr =
             computeMemoryMapOffset( MMIO_GROUP0_CHIP0_XSCOM_BASE_ADDR,
                                     i_node,
                                     l_chipId );
 
-    uint64_t l_xscomAddr = ( (i_scomAddr & XSCOM_MULTICAST_REL_ADDR_MASK) |
-                             XSCOM_MULTICAST |
-                             XSCOM_MULTICAST_OP_READ_OR |
-                             XSCOM_MULTICAST_GROUP_CORE );
 
+    //Map xscom base into processor space
     uint64_t * l_virtAddr =
             static_cast<uint64_t*>
             (mmio_dev_map(reinterpret_cast<void*>(l_xscomBaseAddr),
@@ -892,7 +885,7 @@ uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
     {
         errlHndl_t  l_err = xScomDoOp( DeviceFW::READ,
                 l_virtAddr,
-                l_xscomAddr,
+                i_scomAddr,
                 &l_rv,
                 l_rvSize,
                 l_hmer );
@@ -904,17 +897,14 @@ uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
             l_err = nullptr;
 
             TRACFCOMP( g_trac_xscom,
-                    ERR_MRK "readRemoteCoreScomMultiCast()  Read xScom Failed: "
+                    ERR_MRK "readRemoteScom()  Read xScom Failed: "
                     "XscomAddr = %.16llx, VAddr=%llx",
-                    l_xscomAddr, l_virtAddr );
+                    i_scomAddr, l_virtAddr );
 
             // re-seed return value in case changed before error detected
             l_rv = IPC_INVALID_REMOTE_ADDR | i_node;
             break;
         }
-        else
-        {
-        }
 
         // regs not yet populated
         if (l_rv == 0 )