Fixes to node IPC messaging to handle non-zero base addresses

Current code has each Node calculate each Remote Node's IPC area
remote address by performing a fixed format calculation.
This change has each Node calculating its IPC area Remote address
and posting this value to a local SCOM register.  A Node reads
a Remote Node's SCOM register to acquire the Remote IPC area
address.

Change-Id: I25260ce180e0d07e5e81990d4c1f99e249912491
RTC:191463
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59177
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

18 files changed, 541 insertions, 49 deletions

diff --git a/src/include/kernel/ipc.H b/src/include/kernel/ipc.H
index eeeea2d4b..a29d982a1 100644
--- a/src/include/kernel/ipc.H
+++ b/src/include/kernel/ipc.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -26,6 +26,7 @@
 #define __IPC_H
 
 #include <sys/msg.h>
+#include <sys/internode.h>
 
 #define IPC_DATA_AREA_LOCKED 0xFFFFFFFFFFFFFFFFul
 #define IPC_DATA_AREA_READ   0xFFFFFFFFFFFFFFFEul
@@ -33,12 +34,48 @@
 
 namespace KernelIpc
 {
+
+    /** @fn send()
+     *  @brief Send an IPC message to a remote node
+     *
+     *  @param[in] i_q - message queue
+     *  @param[in] i_msg - message
+     *
+     *  @note
+     */
+    int send(uint64_t i_q, msg_t * i_msg);
+
+    /** @fn updateRemoteIpcAddr()
+     *  @brief update the Remote Address used to send IPC
+     *         messages to the specified node
+     *
+     *  @param[in] i_node - node
+     *  @param[in] i_remoteAddr - IPC address of Remote Node's ipc_data_area
+     *
+     *  @note
+     */
+    int updateRemoteIpcAddr(uint64_t i_node, uint64_t i_remoteAddr);
+
+    /** @fn qryLocalIpcInfo()
+     *  @brief query the node and remote address other nodes will use to
+     *         send messages to this IPC instance.
+     *
+     *  @param[in] i_pONode - buffer the node will be returned in
+     *  @param[in] i_pOAddr - buffer the address will be returned in
+     *
+     *  @note
+     */
+    int qryLocalIpcInfo(uint64_t * i_pONode, uint64_t * i_pOAddr);
+
+
 struct ipc_data_area_t
 {
     uint64_t msg_queue_id;
     msg_t    msg_payload;
     uint64_t hrmor_base;
     uint32_t pir;
+    struct ipc_data_area_t *
+      remote_ipc_data_addr[internode_info_vals_t::MAX_NODES_PER_SYS];
 };
 
 /**
diff --git a/src/include/kernel/syscalls.H b/src/include/kernel/syscalls.H
index c7a03d6b6..c606ad771 100644
--- a/src/include/kernel/syscalls.H
+++ b/src/include/kernel/syscalls.H
@@ -126,7 +126,13 @@ namespace Systemcalls
             /** set_mchk_data() */
         MISC_SETMCHKDATA,
 
-	SYSCALL_MAX
+            /** updateRemoteIpcAddr() */
+        UPDATE_REMOTE_IPC_ADDR,
+
+            /** qryLocalIpcInfo() */
+        QRY_LOCAL_IPC_INFO,
+
+        SYSCALL_MAX
     };
 
     /** @enum SysCalls_FastPath
diff --git a/src/include/sys/internode.h b/src/include/sys/internode.h
index 5581c1e6d..8db0252f6 100644
--- a/src/include/sys/internode.h
+++ b/src/include/sys/internode.h
@@ -26,7 +26,7 @@
 #define __INTERNODE_H
 // Memory area preserved on MPIPL
 
-#include <vmmconst.h>
+#include <usr/vmmconst.h>
 
 enum internode_info_vals_t
 {
diff --git a/src/include/sys/mmio.h b/src/include/sys/mmio.h
index 71340973a..75d923e41 100644
--- a/src/include/sys/mmio.h
+++ b/src/include/sys/mmio.h
@@ -93,8 +93,12 @@ enum MMIO_Scratch_Register
         /** Identifies where hostboot currently resides and how large the
          *  space is */
     MMIO_SCRATCH_MEMORY_STATE = 0x08,
-        /** Spare scratch reg */
-    MMIO_SCRATCH_SPARE = 0x10,
+        /** The address of the Local Node's ipc_data_area buffer in
+         *  the Remote Node's radix.
+         *  The Local Node and the Remote Node use different absolute
+         *  64 bit values to access the same ipc_data_area buffer.
+         *  This register contains the value the Remote Node needs to use. */
+    MMIO_SCRATCH_IPC_DATA_ADDR = 0x10,
         /** Identifies if Hostboot is active after host_start_payload. */
     MMIO_SCRATCH_HOSTBOOT_ACTIVE = 0x18,
 };
diff --git a/src/include/sys/msg.h b/src/include/sys/msg.h
index 2b7f2a8f4..94fda6891 100644
--- a/src/include/sys/msg.h
+++ b/src/include/sys/msg.h
@@ -284,6 +284,35 @@ int msg_respond(msg_q_t q, msg_t* msg);
 msg_t* msg_wait(msg_q_t q);
 
 
+/** @fn updateRemoteIpcAddr
+  * @brief Update a Remote Node's IPC buffer address
+  *
+  * @param[in] i_node - node
+  * @param[in] i_remoteAddr - Remote Node's IPC buffer address
+  *
+  * @return Result of update.
+  * @retval 0 - Success
+  * @retval EINVAL - Invalid Node.
+  */
+           // low nibble is remote node number
+#define IPC_INVALID_REMOTE_ADDR_MASK     0xFFFFFFFFFFFFFFF0ull
+#define IPC_INVALID_REMOTE_ADDR          0x00000000deadadd0ull
+
+int updateRemoteIpcAddr(uint64_t i_node, uint64_t i_remoteAddr);
+
+
+/** @fn qryLocalIpcInfo
+  * @brief Query the local Node's node number and IPC bfr address
+  *
+  * @param[out] o_node - returned Node
+  * @param[out] o_addr - returned Local Node's IPC bfr address
+  *
+  * @return Result of query.
+  * @retval 0 - Success
+   */
+int qryLocalIpcInfo( uint64_t & o_node, uint64_t & o_addr);
+
+
 /** @fn msg_is_async
   * @brief  Indicates if message is asynchronous.
   *
diff --git a/src/include/usr/xscom/xscomif.H b/src/include/usr/xscom/xscomif.H
index 65b7410f1..e547c4621 100644
--- a/src/include/usr/xscom/xscomif.H
+++ b/src/include/usr/xscom/xscomif.H
@@ -35,7 +35,7 @@ namespace XSCOM
 uint64_t get_master_bar( void );
 
 /**
- * @brief Generate a fully-qualified MMIO address for a physical scom
+ *  @brief Generate a fully-qualified MMIO address for a physical scom
  *        address, relative to the given processor target
  *  @param[in] i_proc - Processor
  *  @param[in] i_scomAddr - Physical scom address to convert
@@ -44,6 +44,15 @@ uint64_t get_master_bar( void );
 uint64_t generate_mmio_addr( TARGETING::Target* i_proc,
                              uint64_t i_scomAddr );
 
+/**
+ *  @brief Multicast Read of core XSCOM register on remote Node
+ *  @param[in] i_node - logical Node Number
+ *  @param[in] i_scomAddr - Physical scom address to read
+ *  @return uint64_t - Register value
+ */
+uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
+                                      uint64_t i_scomAddr );
+
 };  // namespace XSCOM
 
 #endif  // end __XSCOMIF_H
diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C
index f097f83da..5123d0b94 100644
--- a/src/kernel/ipc.C
+++ b/src/kernel/ipc.C
@@ -32,11 +32,6 @@
 
 using namespace KernelIpc;
 
-namespace KernelIpc
-{
-    int send(uint64_t i_q, msg_t * i_msg);
-};
-
 /**
  * IPC communication area. Interrupt service provider initializes.
  * @see intrrp.C
@@ -49,28 +44,24 @@ KernelIpc::ipc_data_area_t KernelIpc::ipc_data_area;
 //    2. The destination node never responds, potentially hanging this thread.
 int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
 {
-    // @note
-    // Point to memory in the destination image.
-    // All host boot images are assured to be at the same code level.
-    // PIR node and physical node are not always the same. For instance a
-    // single node system with an alt-master could be designed such that the
-    // alt-master were on a different logical (power bus numbering) node.
-    // Since it's not in plan to use this IPC mechanism in a single node
-    // system, this case will get ignored for now.
-    uint64_t this_node = getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
-    uint64_t hrmor_offset = getHRMOR()-(this_node*(ipc_data_area.hrmor_base));
-    uint64_t dest_node = (i_q >> 32) & 0x07;
-    uint64_t dest_hrmor = (ipc_data_area.hrmor_base*dest_node) + hrmor_offset;
-
-    uint64_t dest_addr = reinterpret_cast<uint64_t>(&ipc_data_area);
-    dest_addr += dest_hrmor;
-    dest_addr |= 0x8000000000000000ul;
-
-    printkd("IPC Dest addr %lx Q_id:%lx\n",dest_addr,i_q);
-
-    // pointer to the ipc_data_area in the destination node
-    ipc_data_area_t * p_dest =
-        reinterpret_cast<ipc_data_area_t*>(dest_addr);
+    // the destination node is a 3 bit field encoded in the 64 bit queue id
+    // big endian bits 29:31, ie, xxxx_xxxN__xxxx_xxxx
+    // extract it from the appropriate field
+    uint64_t dest_node = ((i_q >> 32) &
+                         (internode_info_vals_t::MAX_NODES_PER_SYS - 1));
+
+    ipc_data_area_t * p_dest = ipc_data_area.remote_ipc_data_addr[dest_node];
+    printkd("IPC Dest addr %px Q_id:%lx dest_node:%.lx\n",
+            p_dest,i_q, dest_node);
+
+    // validate destination address
+    if ( (p_dest == nullptr ) ||
+         ((reinterpret_cast<uint64_t>(p_dest) &
+                 IPC_INVALID_REMOTE_ADDR_MASK) ==
+           IPC_INVALID_REMOTE_ADDR))
+    {
+        return( EINVAL);
+    }
 
     // get lock on IPC data area in other node
     if(false == __sync_bool_compare_and_swap(&(p_dest->msg_queue_id),
@@ -100,3 +91,50 @@ int KernelIpc::send(uint64_t i_q, msg_t * i_msg)
     return 0;
 }
 
+
+// update the address this IPC instance will use to send messages
+//  to a remote node
+int KernelIpc::updateRemoteIpcAddr(uint64_t i_Node, uint64_t i_RemoteAddr)
+{
+    int rc;
+    if // input node is valid
+      (i_Node < internode_info_vals_t::MAX_NODES_PER_SYS)
+    {
+        // update local array entry
+        rc = 0;
+        ipc_data_area.remote_ipc_data_addr[i_Node] =
+                reinterpret_cast<ipc_data_area_t*>(i_RemoteAddr);
+    }
+    else
+    {
+        rc = EINVAL;
+        printk("updateRemoteAddr() Invalid input node: %lx\n",i_Node);
+    }
+
+    return(rc);
+}
+
+
+// query the node and remote address other nodes will use to send
+//  messages to this IPC instance
+int KernelIpc::qryLocalIpcInfo(uint64_t * i_pONode, uint64_t * i_pOAddr)
+{
+    // determine node and remote address
+    uint64_t l_localNode = getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE;
+
+    uint64_t l_localAddr = reinterpret_cast<uint64_t>(&ipc_data_area);
+    uint64_t l_hrmorOffset = (getHRMOR() -
+                              (l_localNode * (ipc_data_area.hrmor_base)));
+    uint64_t l_remoteHrmor =
+            ((ipc_data_area.hrmor_base * l_localNode) + l_hrmorOffset);
+
+    uint64_t l_oAddr = (( l_localAddr +
+                          l_remoteHrmor ) |
+                          0x8000000000000000ul);
+
+    *i_pONode = l_localNode;
+    *i_pOAddr = l_oAddr;
+
+    return(0);
+}
+
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index 9dc6bd720..13f6288fd 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -46,11 +46,8 @@
 #include <sys/sync.h>
 #include <errno.h>
 #include <kernel/machchk.H>
+#include <kernel/ipc.H>
 
-namespace KernelIpc
-{
-    int send(uint64_t i_q, msg_t * i_msg);
-};
 
 extern "C"
 void kernel_execute_hype_doorbell()
@@ -144,6 +141,8 @@ namespace Systemcalls
     void MmLinearMap(task_t *t);
     void CritAssert(task_t *t);
     void SetMchkData(task_t *t);
+    void UpdateRemoteIpcAddr(task_t *t);
+    void QryLocalIpcInfo(task_t *t);
 
 
     syscall syscalls[] =
@@ -188,8 +187,10 @@ namespace Systemcalls
         &MmLinearMap,     // MM_LINEAR_MAP
         &CritAssert,  // MISC_CRITASSERT
         &SetMchkData,  // MISC_SETMCHKDATA
+        &UpdateRemoteIpcAddr, // UPDATE_REMOTE_IPC_ADDR
+        &QryLocalIpcInfo  // QRY_LOCAL_IPC_INFO
 
-        };
+    };
 };
 
 extern "C"
@@ -1004,5 +1005,30 @@ namespace Systemcalls
         Kernel::MachineCheck::setCheckstopData(i_xstopAddr,i_xstopData);
     }
 
+    /**
+     *  @brief Tells the kernel the address of the Remote IPC buffer
+     *         for a given Node
+     * @param[in] t: the task calling the update function
+     */
+    void UpdateRemoteIpcAddr(task_t* t)
+    {
+        uint64_t i_Node = TASK_GETARG0(t);
+        uint64_t i_RemoteAddr = TASK_GETARG1(t);
+        int rc = KernelIpc::updateRemoteIpcAddr(i_Node, i_RemoteAddr);
+        TASK_SETRTN(t, rc);
+    }
+
+    /**
+     *  @brief Query the local node and IPC buffer info
+     * @param[in] t: the task calling the update function
+     */
+    void QryLocalIpcInfo(task_t* t)
+    {
+        uint64_t * i_pONode = (uint64_t *)TASK_GETARG0(t);
+        uint64_t * i_pOAddr = (uint64_t *)TASK_GETARG1(t);
+        int rc = KernelIpc::qryLocalIpcInfo(i_pONode, i_pOAddr);
+        TASK_SETRTN(t, rc);
+    }
+
 };
 
diff --git a/src/lib/syscall_msg.C b/src/lib/syscall_msg.C
index e5a63920f..9004461c4 100644
--- a/src/lib/syscall_msg.C
+++ b/src/lib/syscall_msg.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -151,3 +151,31 @@ msg_t* msg_wait(msg_q_t q)
     return (msg_t*)_syscall1(MSG_WAIT, q);
 }
 
+int updateRemoteIpcAddr(uint64_t i_Node, uint64_t i_RemoteAddr)
+{
+    return (int64_t)_syscall2(UPDATE_REMOTE_IPC_ADDR,
+                              (void *)i_Node,
+                              (void *)i_RemoteAddr);
+}
+
+int qryLocalIpcInfo(uint64_t & o_Node, uint64_t & o_Addr)
+{
+    // need to allocate a buffer on the heap as Kernel cannot
+    //  store back to user stack buffer
+    uint64_t * l_pTmpBfr = new uint64_t[2];
+    l_pTmpBfr[0] = o_Node;    // seed tmp bfr with contents of out bfr
+    l_pTmpBfr[1] = o_Addr;    //  to hide the existence of the
+                              //  intermediate tmp bfr
+    void * l_pNode = reinterpret_cast<void *>(&l_pTmpBfr[0]);
+    void * l_pAddr = reinterpret_cast<void *>(&l_pTmpBfr[1]);
+
+    void * l_retVal = _syscall2( QRY_LOCAL_IPC_INFO,
+                                 l_pNode,
+                                 l_pAddr );
+
+    o_Node = l_pTmpBfr[0];
+    o_Addr = l_pTmpBfr[1];
+    delete [] l_pTmpBfr;
+
+    return( reinterpret_cast<int64_t>(l_retVal) );
+}
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 35c7568f9..8e18d2177 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -84,7 +84,7 @@
 #include <secureboot/service.H>
 #include <p9_perst_phb.H>
 #include <plat_hwp_invoker.H>
-
+#include <ipcSp.H>
 // ---------------------------
 // Used to grab SBE boot side
 #include <sbe/sbe_update.H>
@@ -2182,6 +2182,11 @@ void IStepDispatcher::handleProcFabIovalidMsg(msg_t * & io_pMsg)
                    "Returned from cpu_all_winkle." );
             }
 
+            // identify IPC msg address to remote node(s)
+            // this must be after the winkle call because the data is stored
+            // in a core scom which will get lost
+            IPC::IpcSp::distributeLocalNodeAddr();
+
             err = MBOX::resume();
             if (err)
             {
diff --git a/src/usr/initservice/istepdispatcher/makefile b/src/usr/initservice/istepdispatcher/makefile
index 52becfd3a..9726f1775 100644
--- a/src/usr/initservice/istepdispatcher/makefile
+++ b/src/usr/initservice/istepdispatcher/makefile
@@ -35,6 +35,7 @@ EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/procedures/hwp/ffdc/
 EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
 EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/procedures/hwp/nest/
 EXTRAINCDIR += ${ROOTPATH}/src/usr/isteps
+EXTRAINCDIR += ${ROOTPATH}/src/usr/mbox/
 
 VPATH = ${ROOTPATH}/src/usr/initservice/bootconfig/
 
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index fda137259..a7ebde8d5 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -359,25 +359,58 @@ errlHndl_t IntrRp::_init()
             break;
         }
 
+        // extract the mpipl indicator
+        // will still be 0 if attribute does not exist
         uint8_t is_mpipl = 0;
-        TARGETING::Target * sys = NULL;
-        TARGETING::targetService().getTopLevelTarget(sys);
-        if(sys &&
-            sys->tryGetAttr<TARGETING::ATTR_IS_MPIPL_HB>(is_mpipl) &&
-            is_mpipl)
+        sys->tryGetAttr<TARGETING::ATTR_IS_MPIPL_HB>(is_mpipl);
+
+        // Extract the last values for IPC data Addresses.
+        // will be 00's except for MPIPL, then previous values
+        uint64_t l_ipcDataAddrs[MAX_NODES_PER_SYS];
+        assert((sys->tryGetAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+        (l_ipcDataAddrs)) == true );
+
+        // determine node and
+        // ipc data address as seen by a remote node
+        uint64_t l_thisNode = 0xa5a5;   // seed with invalid value(s) to
+        uint64_t l_remoteAddr = 0x5a5a; // catch missing return values
+
+        qryLocalIpcInfo( l_thisNode, l_remoteAddr );
+
+        // validate no change to local if this is an MPIPL
+        if (is_mpipl)
+        {
+            assert( l_ipcDataAddrs[l_thisNode] == l_remoteAddr );
+        }
+
+        // update attribute entry for this node
+        l_ipcDataAddrs[l_thisNode] = l_remoteAddr;
+        sys->setAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+        (l_ipcDataAddrs);
+
+        // shadow the IPC addrs where the IPC msg send can reach them
+        for ( uint64_t i = 0;
+                i < MAX_NODES_PER_SYS;
+                i++ )
+        {
+            uint64_t remoteAddr = l_ipcDataAddrs[i];
+            updateRemoteIpcAddr( i, remoteAddr );
+        }
+
+        if(is_mpipl)
         {
             TRACFCOMP(g_trac_intr,"Reset interrupt service for MPIPL");
             l_err = resetIntpForMpipl();
 
             if(l_err)
             {
-                TRACFCOMP(g_trac_intr,"Failed to reset interrupt service for MPIPL");
+                TRACFCOMP(g_trac_intr,
+                        "Failed to reset interrupt service for MPIPL");
                 break;
             }
         }
 
 
-
         //Disable Incoming PSI Interrupts
         TRACDCOMP(g_trac_intr, "IntrRp::_init() Disabling PSI Interrupts");
         uint64_t l_disablePsiIntr = PSI_BRIDGE_INTP_STATUS_CTL_DISABLE_PSI;
diff --git a/src/usr/isteps/istep16/call_host_ipl_complete.C b/src/usr/isteps/istep16/call_host_ipl_complete.C
index 3993fda8b..a96703191 100644
--- a/src/usr/isteps/istep16/call_host_ipl_complete.C
+++ b/src/usr/isteps/istep16/call_host_ipl_complete.C
@@ -50,6 +50,7 @@
 #ifdef CONFIG_BMC_IPMI
 #include    <ipmi/ipmisensor.H>
 #endif
+
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   ISTEP;
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index d0110db98..61d9f846d 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -35,6 +35,12 @@
 #include <sbeio/sbeioif.H>
 #include <util/utiltce.H>
 #include <targeting/targplatutil.H>
+#include <targeting/common/targetservice.H>
+#include <targeting/common/attributes.H>
+#include <p9_quad_scom_addresses.H>
+#include <sys/internode.h>
+#include <sys/mmio.h>
+#include <xscom/xscomif.H>
 
 namespace ISTEP_21
 {
@@ -54,7 +60,8 @@ using namespace TARGETING;
 
 IpcSp::IpcSp()
     :
-        iv_msgQ()
+        iv_msgQ(),
+        iv_IsRemoteNodeAddrsValid( false )
 {
 }
 
@@ -68,6 +75,48 @@ void IpcSp::init(errlHndl_t & o_errl)
     o_errl = Singleton<IpcSp>::instance()._init();
 }
 
+void IpcSp::distributeLocalNodeAddr( void )
+{
+    // Store IPC address for local node in core scratch registers
+    //  to identify IPC msg address to remote node(s)
+    uint64_t l_localNode;
+    uint64_t l_remoteAddr;
+    qryLocalIpcInfo( l_localNode, l_remoteAddr );
+
+    TARGETING::Target * l_pSys = NULL;
+    TARGETING::targetService().getTopLevelTarget( l_pSys );
+    TARGETING::TargetHandleList l_coreTargetList;
+    TARGETING::getChildChiplets( l_coreTargetList,
+                                 l_pSys,
+                                 TARGETING::TYPE_CORE,
+                                 true );
+
+    // Store IPC address into scom reg for each core
+    // Every core on this node needs to have the value stored
+    //  in it's scratch register in case any cores get deconfigured
+    for(const auto & l_core_target : l_coreTargetList)
+    {
+        uint64_t l_remoteAddrSize = sizeof(l_remoteAddr);
+        errlHndl_t l_err = deviceWrite( l_core_target,
+                                        &l_remoteAddr,
+                                        l_remoteAddrSize,
+                                        DEVICE_SCOM_ADDRESS(C_SCR2) );
+
+        if (l_err)
+        {
+            errlCommit(l_err, IPC_COMP_ID);
+        }
+    }
+
+    return;
+}
+
+void IpcSp::acquireRemoteNodeAddrs( void )
+{
+    Singleton<IpcSp>::instance()._acquireRemoteNodeAddrs();
+    return;
+}
+
 void* IpcSp::msg_handler(void *unused)
 {
     Singleton<IpcSp>::instance().msgHandler();
@@ -586,3 +635,88 @@ void IpcSp::msgHandler()
 }
 
 
+void IpcSp::_acquireRemoteNodeAddrs( void )
+{
+    if // remote addresses have not been gathered
+      (iv_IsRemoteNodeAddrsValid == false)
+    {
+        TARGETING::Target * l_sys = NULL;
+        TARGETING::targetService().getTopLevelTarget(l_sys);
+
+        if (l_sys)
+        {
+            // If MPIPL then remote regs are no longer valid
+            uint8_t l_IsMpipl =
+              l_sys->getAttr<TARGETING::ATTR_IS_MPIPL_HB>();
+
+            if (l_IsMpipl == false )
+            {
+                // extract current ipc addr attributes
+                uint64_t l_ipcDataAddrs[MAX_NODES_PER_SYS];
+                l_sys->tryGetAttr
+                <TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+                (l_ipcDataAddrs);
+
+                // extract valid node map
+                uint8_t validNodeBitMap =
+                        l_sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>();
+
+                // determine which node is local
+                uint64_t l_ThisNode;
+                uint64_t l_RemoteAddr;
+                qryLocalIpcInfo( l_ThisNode, l_RemoteAddr );
+
+                // loop thru all Nodes
+                for ( uint64_t i = 0;
+                      i < MAX_NODES_PER_SYS;
+                      i++ )
+                {
+                    if // remote node
+                      ( i != l_ThisNode )
+                    {
+                        if // valid node
+                          ( (validNodeBitMap & (0x80 >> i)) != 0  )
+                        {
+                            // read scoms for remote node
+                            l_RemoteAddr =
+                                    XSCOM::readRemoteCoreScomMultiCast(i,
+                                                                       C_SCR2);
+                        } // end valid node
+                        else
+                        {
+                            // set an invalid value for remote address
+                            l_RemoteAddr = IPC_INVALID_REMOTE_ADDR | i;
+                        }
+
+                        // push results to IPC and Attributes shadow
+                        updateRemoteIpcAddr( i, l_RemoteAddr );
+                        l_ipcDataAddrs[i] = l_RemoteAddr;
+                    } // end remote node
+                    else
+                    {
+                        // local node, do not need to get remote addrs
+                    }
+                }
+
+                // update attributes
+                l_sys->setAttr<TARGETING::ATTR_IPC_NODE_BUFFER_GLOBAL_ADDRESS>
+                (l_ipcDataAddrs);
+            } // end not mpipl
+            else
+            {
+                // (remote addrs have already been shadowed from attributes)
+            }
+
+            // read the scoms only once
+            iv_IsRemoteNodeAddrsValid = true;
+        } // end acquire scoms
+    } // end remote addrs not gathered
+    else
+    {
+        // (no action needed)
+    }
+
+    return;
+}
+
+
diff --git a/src/usr/mbox/ipcSp.H b/src/usr/mbox/ipcSp.H
index 1a3ce0045..009ad2be2 100644
--- a/src/usr/mbox/ipcSp.H
+++ b/src/usr/mbox/ipcSp.H
@@ -44,6 +44,18 @@ namespace IPC
              */
             static void init(errlHndl_t & o_errl);
 
+            /**
+             * Query the local node's IPC address and store it all of the
+             * local node's core scratch2 regs. This will allow remote nodes
+             * to read the address so they know where to write IPC messages.
+             */
+            static void distributeLocalNodeAddr( void );
+
+            /**
+             * Acquire all Remote Node IPC addresses
+             */
+            static void acquireRemoteNodeAddrs( void );
+
         protected:
 
             /**
@@ -77,10 +89,21 @@ namespace IPC
             void msgHandler();
 
             /**
+             * Acquire all Remote Node IPC addresses
+             */
+            void _acquireRemoteNodeAddrs( void );
+
+            /**
              * The IPC message queue
              */
             msg_q_t iv_msgQ;
 
+            /**
+             * Indicates whether or not the Remote Node IPC
+             * addresses have already been acquired
+             */
+            bool iv_IsRemoteNodeAddrsValid;
+
     };
 }; // namespace IPC
 #endif
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index c0b63c3f1..777699e34 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -2138,6 +2138,9 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
             }
             else
             {
+                // stitch together routing if not already done
+                IPC::IpcSp::acquireRemoteNodeAddrs();
+
                 int rc = msg_send(reinterpret_cast<msg_q_t>(q_handle),
                                   i_msg);
                 TRACFCOMP(g_trac_mboxmsg,INFO_MRK
diff --git a/src/usr/mbox/makefile b/src/usr/mbox/makefile
index 96d750804..1019e28e2 100644
--- a/src/usr/mbox/makefile
+++ b/src/usr/mbox/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2012,2014
+# Contributors Listed Below - COPYRIGHT 2012,2018
 # [+] International Business Machines Corp.
 #
 #
@@ -25,6 +25,8 @@
 ROOTPATH = ../../..
 MODULE = mbox
 
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
+
 OBJS += mboxdd.o
 OBJS += mailboxsp.o
 OBJS += mbox_dma_buffer.o
diff --git a/src/usr/xscom/xscom.C b/src/usr/xscom/xscom.C
index 18594d7cd..7c8bd607c 100644
--- a/src/usr/xscom/xscom.C
+++ b/src/usr/xscom/xscom.C
@@ -35,12 +35,14 @@
 #include <sys/task.h>
 #include <sys/sync.h>
 #include <sys/misc.h>
+#include <sys/time.h>
 #include <string.h>
 #include <devicefw/driverif.H>
 #include <trace/interface.H>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <targeting/common/targetservice.H>
+#include <targeting/common/utilFilter.H>
 #include <xscom/xscomreasoncodes.H>
 #include "xscom.H"
 #include <assert.h>
@@ -818,4 +820,115 @@ uint64_t generate_mmio_addr( TARGETING::Target* i_proc,
     return l_returnAddr;
 }
 
+
+/**
+ * @brief Multicast Read of core XSCOM register on remote Node
+ */
+uint64_t readRemoteCoreScomMultiCast( uint64_t i_node,
+                                      uint64_t i_scomAddr )
+{
+    // definitions of 64 bit xscom address contents that are
+    //   useful for this function
+    //                             hi 32 bits  lo 32 bits
+    //  representation :          0x0000_0000__0000_0000
+    //
+    //           (hi 32 bits, xscom mmio control)
+    //
+    //  chip field                0x0000_0400__0000_0000
+    //    (to be multiplied by chip ID to give field value)
+    //
+    //  node/group field (0-7)    0xE000_0000__0000_0000
+    //
+    //           (lo 32 bits, scom addr)
+    //
+    //  multicast                 0x0000_0000__4000_0000
+    //  multicast op type         0x0000_0000__3800_0000
+    //    - read OR                            00
+    //    - read AND                           08
+    //    - read bitwise                       10
+    //    - rsvd                               18
+    //    - write compare                      20
+    //    - write                              28
+    //    - rsvd                               30
+    //    - rsvd                               38
+    //  multicast group           0x0000_0000__0700_0000
+    //  relative scomAddr field   0x0000_0000__00FF_FFFF
+    constexpr uint64_t XSCOM_MULTICAST =                  0x0000000040000000;
+    constexpr uint64_t XSCOM_MULTICAST_OP_READ_OR =       0x0000000000000000;
+    constexpr uint64_t XSCOM_MULTICAST_GROUP_CORE =       0x0000000001000000;
+    constexpr uint64_t XSCOM_MULTICAST_REL_ADDR_MASK =    0x0000000000FFFFFF;
+
+    // Symmetry between nodes is enforced so we know the remote
+    //  node contains this chip
+    TARGETING::Target * l_MasterProcTarget = nullptr;
+    TARGETING::TargetService & l_tgtServ = TARGETING::targetService();
+    l_tgtServ.masterProcChipTargetHandle( l_MasterProcTarget );
+    uint8_t l_chipId =
+            l_MasterProcTarget->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
+
+    // compute xscom address & control, then map into processor space
+    uint64_t l_xscomBaseAddr =
+            computeMemoryMapOffset( MMIO_GROUP0_CHIP0_XSCOM_BASE_ADDR,
+                                    i_node,
+                                    l_chipId );
+
+    uint64_t l_xscomAddr = ( (i_scomAddr & XSCOM_MULTICAST_REL_ADDR_MASK) |
+                             XSCOM_MULTICAST |
+                             XSCOM_MULTICAST_OP_READ_OR |
+                             XSCOM_MULTICAST_GROUP_CORE );
+
+    uint64_t * l_virtAddr =
+            static_cast<uint64_t*>
+            (mmio_dev_map(reinterpret_cast<void*>(l_xscomBaseAddr),
+                          THIRTYTWO_GB));
+
+    // execute the SCOM op to do the multicast read
+    // init return value to dummy to verify read happened
+    uint64_t l_rv = IPC_INVALID_REMOTE_ADDR | i_node;
+    size_t l_rvSize = sizeof(l_rv);
+    HMER l_hmer;
+
+    do
+    {
+        errlHndl_t  l_err = xScomDoOp( DeviceFW::READ,
+                l_virtAddr,
+                l_xscomAddr,
+                &l_rv,
+                l_rvSize,
+                l_hmer );
+
+        // If not successful
+        if (l_err)
+        {
+            delete l_err;
+            l_err = nullptr;
+
+            TRACFCOMP( g_trac_xscom,
+                    ERR_MRK "readRemoteCoreScomMultiCast()  Read xScom Failed: "
+                    "XscomAddr = %.16llx, VAddr=%llx",
+                    l_xscomAddr, l_virtAddr );
+
+            // re-seed return value in case changed before error detected
+            l_rv = IPC_INVALID_REMOTE_ADDR | i_node;
+            break;
+        }
+        else
+        {
+        }
+
+        // regs not yet populated
+        if (l_rv == 0 )
+        {
+          // delay to allow remote core to populate regs
+          nanosleep(0, (NS_PER_MSEC * 100));
+        }
+
+        // wait for remote node to populate its core scom regs
+    } while( l_rv == 0 );
+
+    mmio_dev_unmap(reinterpret_cast<void*>(l_virtAddr));
+
+    return l_rv;
+}
+
 } // end namespace