Fixes to node IPC messaging to handle non-zero base addresses

Current code has each Node calculate each Remote Node's IPC area
remote address by performing a fixed format calculation.
This change has each Node calculating its IPC area Remote address
and posting this value to a local SCOM register.  A Node reads
a Remote Node's SCOM register to acquire the Remote IPC area
address.

Change-Id: I25260ce180e0d07e5e81990d4c1f99e249912491
RTC:191463
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59177
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

9 files changed, 325 insertions, 10 deletions

diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 35c7568f9..8e18d2177 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -84,7 +84,7 @@
 #include <secureboot/service.H>
 #include <p9_perst_phb.H>
 #include <plat_hwp_invoker.H>
-
+#include <ipcSp.H>
 // ---------------------------
 // Used to grab SBE boot side
 #include <sbe/sbe_update.H>
@@ -2182,6 +2182,11 @@ void IStepDispatcher::handleProcFabIovalidMsg(msg_t * & io_pMsg)
                    "Returned from cpu_all_winkle." );
             }
 
+            // identify IPC msg address to remote node(s)
+            // this must be after the winkle call because the data is stored
+            // in a core scom which will get lost
+            IPC::IpcSp::distributeLocalNodeAddr();
+
             err = MBOX::resume();
             if (err)
             {
diff --git a/src/usr/initservice/istepdispatcher/makefile b/src/usr/initservice/istepdispatcher/makefile
index 52becfd3a..9726f1775 100644
--- a/src/usr/initservice/istepdispatcher/makefile
+++ b/src/usr/initservice/istepdispatcher/makefile
@@ -35,6 +35,7 @@ EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/procedures/hwp/ffdc/
 EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
 EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/procedures/hwp/nest/
 EXTRAINCDIR += ${ROOTPATH}/src/usr/isteps
+EXTRAINCDIR += ${ROOTPATH}/src/usr/mbox/
 
 VPATH = ${ROOTPATH}/src/usr/initservice/bootconfig/
 
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index fda137259..a7ebde8d5 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -359,25 +359,58 @@ errlHndl_t IntrRp::_init()
             break;
         }
 
+        // extract the mpipl indicator
+        // will still be 0 if attribute does not exist
         uint8_t is_mpipl = 0;
-        TARGETING::Target * sys = NULL;
-        TARGETING::targetService().getTopLevelTarget(sys);
-        if(sys &&
-            sys->tryGetAttr<TARGETING::ATTR_IS_MPIPL_HB>(is_mpipl) &&
-            is_mpipl)
+        sys->tryGetAttr<TARGETING::ATTR_IS_MPIPL_HB>(is_mpipl);
+
+        // Extract the last values for IPC data Addresses.
+        // will be 00's except for MPIPL, then previous values
+        uint64_t l_ipcDataAddrs[MAX_NODES_PER_SYS];
+        assert((sys->tryGetAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+        (l_ipcDataAddrs)) == true );
+
+        // determine node and
+        // ipc data address as seen by a remote node
+        uint64_t l_thisNode = 0xa5a5;   // seed with invalid value(s) to
+        uint64_t l_remoteAddr = 0x5a5a; // catch missing return values
+
+        qryLocalIpcInfo( l_thisNode, l_remoteAddr );
+
+        // validate no change to local if this is an MPIPL
+        if (is_mpipl)
+        {
+            assert( l_ipcDataAddrs[l_thisNode] == l_remoteAddr );
+        }
+
+        // update attribute entry for this node
+        l_ipcDataAddrs[l_thisNode] = l_remoteAddr;
+        sys->setAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+        (l_ipcDataAddrs);
+
+        // shadow the IPC addrs where the IPC msg send can reach them
+        for ( uint64_t i = 0;
+                i < MAX_NODES_PER_SYS;
+                i++ )
+        {
+            uint64_t remoteAddr = l_ipcDataAddrs[i];
+            updateRemoteIpcAddr( i, remoteAddr );
+        }
+
+        if(is_mpipl)
         {
             TRACFCOMP(g_trac_intr,"Reset interrupt service for MPIPL");
             l_err = resetIntpForMpipl();
 
             if(l_err)
             {
-                TRACFCOMP(g_trac_intr,"Failed to reset interrupt service for MPIPL");
+                TRACFCOMP(g_trac_intr,
+                        "Failed to reset interrupt service for MPIPL");
                 break;
             }
         }
 
 
-
         //Disable Incoming PSI Interrupts
         TRACDCOMP(g_trac_intr, "IntrRp::_init() Disabling PSI Interrupts");
         uint64_t l_disablePsiIntr = PSI_BRIDGE_INTP_STATUS_CTL_DISABLE_PSI;
diff --git a/src/usr/isteps/istep16/call_host_ipl_complete.C b/src/usr/isteps/istep16/call_host_ipl_complete.C
index 3993fda8b..a96703191 100644
--- a/src/usr/isteps/istep16/call_host_ipl_complete.C
+++ b/src/usr/isteps/istep16/call_host_ipl_complete.C
@@ -50,6 +50,7 @@
 #ifdef CONFIG_BMC_IPMI
 #include    <ipmi/ipmisensor.H>
 #endif
+
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   ISTEP;
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index d0110db98..61d9f846d 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -35,6 +35,12 @@
 #include <sbeio/sbeioif.H>
 #include <util/utiltce.H>
 #include <targeting/targplatutil.H>
+#include <targeting/common/targetservice.H>
+#include <targeting/common/attributes.H>
+#include <p9_quad_scom_addresses.H>
+#include <sys/internode.h>
+#include <sys/mmio.h>
+#include <xscom/xscomif.H>
 
 namespace ISTEP_21
 {
@@ -54,7 +60,8 @@ using namespace TARGETING;
 
 IpcSp::IpcSp()
     :
-        iv_msgQ()
+        iv_msgQ(),
+        iv_IsRemoteNodeAddrsValid( false )
 {
 }
 
@@ -68,6 +75,48 @@ void IpcSp::init(errlHndl_t & o_errl)
     o_errl = Singleton<IpcSp>::instance()._init();
 }
 
+void IpcSp::distributeLocalNodeAddr( void )
+{
+    // Store IPC address for local node in core scratch registers
+    //  to identify IPC msg address to remote node(s)
+    uint64_t l_localNode;
+    uint64_t l_remoteAddr;
+    qryLocalIpcInfo( l_localNode, l_remoteAddr );
+
+    TARGETING::Target * l_pSys = NULL;
+    TARGETING::targetService().getTopLevelTarget( l_pSys );
+    TARGETING::TargetHandleList l_coreTargetList;
+    TARGETING::getChildChiplets( l_coreTargetList,
+                                 l_pSys,
+                                 TARGETING::TYPE_CORE,
+                                 true );
+
+    // Store IPC address into scom reg for each core
+    // Every core on this node needs to have the value stored
+    //  in it's scratch register in case any cores get deconfigured
+    for(const auto & l_core_target : l_coreTargetList)
+    {
+        uint64_t l_remoteAddrSize = sizeof(l_remoteAddr);
+        errlHndl_t l_err = deviceWrite( l_core_target,
+                                        &l_remoteAddr,
+                                        l_remoteAddrSize,
+                                        DEVICE_SCOM_ADDRESS(C_SCR2) );
+
+        if (l_err)
+        {
+            errlCommit(l_err, IPC_COMP_ID);
+        }
+    }
+
+    return;
+}
+
+void IpcSp::acquireRemoteNodeAddrs( void )
+{
+    Singleton<IpcSp>::instance()._acquireRemoteNodeAddrs();
+    return;
+}
+
 void* IpcSp::msg_handler(void *unused)
 {
     Singleton<IpcSp>::instance().msgHandler();
@@ -586,3 +635,88 @@ void IpcSp::msgHandler()
 }
 
 
+void IpcSp::_acquireRemoteNodeAddrs( void )
+{
+    if // remote addresses have not been gathered
+      (iv_IsRemoteNodeAddrsValid == false)
+    {
+        TARGETING::Target * l_sys = NULL;
+        TARGETING::targetService().getTopLevelTarget(l_sys);
+
+        if (l_sys)
+        {
+            // If MPIPL then remote regs are no longer valid
+            uint8_t l_IsMpipl =
+              l_sys->getAttr<TARGETING::ATTR_IS_MPIPL_HB>();
+
+            if (l_IsMpipl == false )
+            {
+                // extract current ipc addr attributes
+                uint64_t l_ipcDataAddrs[MAX_NODES_PER_SYS];
+                l_sys->tryGetAttr
+                <TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+                (l_ipcDataAddrs);
+
+                // extract valid node map
+                uint8_t validNodeBitMap =
+                        l_sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>();
+
+                // determine which node is local
+                uint64_t l_ThisNode;
+                uint64_t l_RemoteAddr;
+                qryLocalIpcInfo( l_ThisNode, l_RemoteAddr );
+
+                // loop thru all Nodes
+                for ( uint64_t i = 0;
+                      i < MAX_NODES_PER_SYS;
+                      i++ )
+                {
+                    if // remote node
+                      ( i != l_ThisNode )
+                    {
+                        if // valid node
+                          ( (validNodeBitMap & (0x80 >> i)) != 0  )
+                        {
+                            // read scoms for remote node
+                            l_RemoteAddr =
+                                    XSCOM::readRemoteCoreScomMultiCast(i,
+                                                                       C_SCR2);
+                        } // end valid node
+                        else
+                        {
+                            // set an invalid value for remote address
+                            l_RemoteAddr = IPC_INVALID_REMOTE_ADDR | i;
+                        }
+
+                        // push results to IPC and Attributes shadow
+                        updateRemoteIpcAddr( i, l_RemoteAddr );
+                        l_ipcDataAddrs[i] = l_RemoteAddr;
+                    } // end remote node
+                    else
+                    {
+                        // local node, do not need to get remote addrs
+                    }
+                }
+
+                // update attributes
+                l_sys->setAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+                (l_ipcDataAddrs);
+            } // end not mpipl
+            else
+            {
+                // (remote addrs have already been shadowed from attributes)
+            }
+
+            // read the scoms only once
+            iv_IsRemoteNodeAddrsValid = true;
+        } // end acquire scoms
+    } // end remote addrs not gathered
+    else
+    {
+        // (no action needed)
+    }
+
+    return;
+}
+
+
diff --git a/src/usr/mbox/ipcSp.H b/src/usr/mbox/ipcSp.H
index 1a3ce0045..009ad2be2 100644
--- a/src/usr/mbox/ipcSp.H
+++ b/src/usr/mbox/ipcSp.H
@@ -44,6 +44,18 @@ namespace IPC
              */
             static void init(errlHndl_t & o_errl);
 
+            /**
+             * Query the local node's IPC address and store it all of the
+             * local node's core scratch2 regs. This will allow remote nodes
+             * to read the address so they know where to write IPC messages.
+             */
+            static void distributeLocalNodeAddr( void );
+
+            /**
+             * Acquire all Remote Node IPC addresses
+             */
+            static void acquireRemoteNodeAddrs( void );
+
         protected:
 
             /**
@@ -77,10 +89,21 @@ namespace IPC
             void msgHandler();
 
             /**
+             * Acquire all Remote Node IPC addresses
+             */
+            void _acquireRemoteNodeAddrs( void );
+
+            /**
              * The IPC message queue
              */
             msg_q_t iv_msgQ;
 
+            /**
+             * Indicates whether or not the Remote Node IPC
+             * addresses have already been acquired
+             */
+            bool iv_IsRemoteNodeAddrsValid;
+
     };
 }; // namespace IPC
 #endif
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index c0b63c3f1..777699e34 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -2138,6 +2138,9 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
             }
             else
             {
+                // stitch together routing if not already done
+                IPC::IpcSp::acquireRemoteNodeAddrs();
+
                 int rc = msg_send(reinterpret_cast<msg_q_t>(q_handle),
                                   i_msg);
                 TRACFCOMP(g_trac_mboxmsg,INFO_MRK
diff --git a/src/usr/mbox/makefile b/src/usr/mbox/makefile
index 96d750804..1019e28e2 100644
--- a/src/usr/mbox/makefile
+++ b/src/usr/mbox/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2012,2014
+# Contributors Listed Below - COPYRIGHT 2012,2018
 # [+] International Business Machines Corp.
 #
 #
@@ -25,6 +25,8 @@
 ROOTPATH = ../../..
 MODULE = mbox
 
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
+
 OBJS += mboxdd.o
 OBJS += mailboxsp.o
 OBJS += mbox_dma_buffer.o
diff --git a/src/usr/xscom/xscom.C b/src/usr/xscom/xscom.C
index 18594d7cd..7c8bd607c 100644
--- a/src/usr/xscom/xscom.C
+++ b/src/usr/xscom/xscom.C
@@ -35,12 +35,14 @@
 #include <sys/task.h>
 #include <sys/sync.h>
 #include <sys/misc.h>
+#include <sys/time.h>
 #include <string.h>
 #include <devicefw/driverif.H>
 #include <trace/interface.H>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <targeting/common/targetservice.H>
+#include <targeting/common/utilFilter.H>
 #include <xscom/xscomreasoncodes.H>
 #include "xscom.H"
 #include <assert.h>
@@ -818,4 +820,115 @@ uint64_t generate_mmio_addr( TARGETING::Target* i_proc,
     return l_returnAddr;
 }
 
+
+/**
+ * @brief Multicast Read of core XSCOM register on remote Node
+ */
+uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
+                                      uint64_t i_scomAddr )
+{
+    // definitions of 64 bit xscom address contents that are
+    //   useful for this function
+    //                             hi 32 bits  lo 32 bits
+    //  representation :          0x0000_0000__0000_0000
+    //
+    //           (hi 32 bits, xscom mmio control)
+    //
+    //  chip field                0x0000_0400__0000_0000
+    //    (to be multiplied by chip ID to give field value)
+    //
+    //  node/group field (0-7)    0xE000_0000__0000_0000
+    //
+    //           (lo 32 bits, scom addr)
+    //
+    //  multicast                 0x0000_0000__4000_0000
+    //  multicast op type         0x0000_0000__3800_0000
+    //    - read OR                            00
+    //    - read AND                           08
+    //    - read bitwise                       10
+    //    - rsvd                               18
+    //    - write compare                      20
+    //    - write                              28
+    //    - rsvd                               30
+    //    - rsvd                               38
+    //  multicast group           0x0000_0000__0700_0000
+    //  relative scomAddr field   0x0000_0000__00FF_FFFF
+    constexpr uint64_t XSCOM_MULTICAST =                  0x0000000040000000;
+    constexpr uint64_t XSCOM_MULTICAST_OP_READ_OR =       0x0000000000000000;
+    constexpr uint64_t XSCOM_MULTICAST_GROUP_CORE =       0x0000000001000000;
+    constexpr uint64_t XSCOM_MULTICAST_REL_ADDR_MASK =    0x0000000000FFFFFF;
+
+    // Symmetry between nodes is enforced so we know the remote
+    //  node contains this chip
+    TARGETING::Target * l_MasterProcTarget = nullptr;
+    TARGETING::TargetService & l_tgtServ = TARGETING::targetService();
+    l_tgtServ.masterProcChipTargetHandle( l_MasterProcTarget );
+    uint8_t l_chipId =
+            l_MasterProcTarget->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
+
+    // compute xscom address & control, then map into processor space
+    uint64_t l_xscomBaseAddr =
+            computeMemoryMapOffset( MMIO_GROUP0_CHIP0_XSCOM_BASE_ADDR,
+                                    i_node,
+                                    l_chipId );
+
+    uint64_t l_xscomAddr = ( (i_scomAddr & XSCOM_MULTICAST_REL_ADDR_MASK) |
+                             XSCOM_MULTICAST |
+                             XSCOM_MULTICAST_OP_READ_OR |
+                             XSCOM_MULTICAST_GROUP_CORE );
+
+    uint64_t * l_virtAddr =
+            static_cast<uint64_t*>
+            (mmio_dev_map(reinterpret_cast<void*>(l_xscomBaseAddr),
+                          THIRTYTWO_GB));
+
+    // execute the SCOM op to do the multicast read
+    // init return value to dummy to verify read happened
+    uint64_t l_rv = IPC_INVALID_REMOTE_ADDR | i_node;
+    size_t l_rvSize = sizeof(l_rv);
+    HMER l_hmer;
+
+    do
+    {
+        errlHndl_t  l_err = xScomDoOp( DeviceFW::READ,
+                l_virtAddr,
+                l_xscomAddr,
+                &l_rv,
+                l_rvSize,
+                l_hmer );
+
+        // If not successful
+        if (l_err)
+        {
+            delete l_err;
+            l_err = nullptr;
+
+            TRACFCOMP( g_trac_xscom,
+                    ERR_MRK "readRemoteCoreScomMultiCast()  Read xScom Failed: "
+                    "XscomAddr = %.16llx, VAddr=%llx",
+                    l_xscomAddr, l_virtAddr );
+
+            // re-seed return value in case changed before error detected
+            l_rv = IPC_INVALID_REMOTE_ADDR | i_node;
+            break;
+        }
+        else
+        {
+        }
+
+        // regs not yet populated
+        if (l_rv == 0 )
+        {
+          // delay to allow remote core to populate regs
+          nanosleep(0, (NS_PER_MSEC * 100));
+        }
+
+        // wait for remote node to populate its core scom regs
+    } while( l_rv == 0 );
+
+    mmio_dev_unmap(reinterpret_cast<void*>(l_virtAddr));
+
+    return l_rv;
+}
+
 } // end namespace