Handle pending interrupts across nodes in mpipl

RTC: 80988
Change-Id: I7ef9feaa3d163d6956576f30538e2fe001e892a1
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/8441
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

9 files changed, 507 insertions, 5 deletions

diff --git a/src/include/sys/internode.h b/src/include/sys/internode.h
new file mode 100644
index 000000000..3d4c6bf65
--- /dev/null
+++ b/src/include/sys/internode.h
@@ -0,0 +1,58 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/include/sys/internode.h $                                 */
+/*                                                                        */
+/* IBM CONFIDENTIAL                                                       */
+/*                                                                        */
+/* COPYRIGHT International Business Machines Corp. 2014                   */
+/*                                                                        */
+/* p1                                                                     */
+/*                                                                        */
+/* Object Code Only (OCO) source materials                                */
+/* Licensed Internal Code Source Materials                                */
+/* IBM HostBoot Licensed Internal Code                                    */
+/*                                                                        */
+/* The source code for this program is not published or otherwise         */
+/* divested of its trade secrets, irrespective of what has been           */
+/* deposited with the U.S. Copyright Office.                              */
+/*                                                                        */
+/* Origin: 30                                                             */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#ifndef __INTERNODE_H
+#define __INTERNODE_H
+// Memory area preserved on MPIPL
+
+#include <vmmconst.h>
+
+enum internode_info_vals_t
+{
+    MAX_NODES_PER_SYS = 8,
+};
+
+enum intr_mpipl_sync_t
+{
+    INTR_MPIPL_SYNC_CLEAR = 0,
+    INTR_MPIPL_UPSTREAM_DISABLED = 1,
+    INTR_MPIPL_DRAINED = 2,
+};
+
+
+/**
+ * Node information needed across MPIPLs
+ */
+struct internode_info_t
+{
+    uint64_t eye_catcher;
+    uint64_t version;
+    bool  exist[ MAX_NODES_PER_SYS ];       //!< true if HB node exists
+    volatile intr_mpipl_sync_t  mpipl_intr_sync;     //!< at sync point
+};
+
+#define NODE_INFO_EYE_CATCHER 0x4e4f444544415441ul // "NODEDATA"
+#define NODE_INFO_VERSION 1
+
+#define INTERNODE_INFO_SIZE (sizeof(internode_info_t))
+
+#endif
diff --git a/src/include/usr/intr/interrupt.H b/src/include/usr/intr/interrupt.H
index f8dd202ac..b8ab621b2 100644
--- a/src/include/usr/intr/interrupt.H
+++ b/src/include/usr/intr/interrupt.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -145,6 +145,7 @@ namespace INTR
         MSG_INTR_ENABLE_PSI_INTR,   //!< Enable PSI interrupts
         MSG_INTR_MPIPL_CLEANUP,     //!< Clean up interrupts on MPIPL
         MSG_INTR_ADD_CPU_TIMEOUT,   //!< Check for a timeout waiting for a core.
+        MSG_INTR_ADD_HBNODE,        //!< Add node info for MPIPL
     };
 
     /**
@@ -212,6 +213,13 @@ namespace INTR
     uint64_t getIntpAddr(const TARGETING::Target * i_ex,
                          uint8_t i_thread);
 
+    /**
+     * Indicate the existance of another HB node in the system
+     * @param[in] The HB instance number (HB node)
+     * @return error log handle on error
+     */
+    errlHndl_t addHbNode(uint64_t i_hbNode);
+
 };
 
 #endif
diff --git a/src/include/usr/intr/intr_reasoncodes.H b/src/include/usr/intr/intr_reasoncodes.H
index 7965875b5..3cdce388d 100644
--- a/src/include/usr/intr/intr_reasoncodes.H
+++ b/src/include/usr/intr/intr_reasoncodes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -37,6 +37,10 @@ namespace INTR
         MOD_INTRRP_REGISTERINTERRUPT = 0x05, /**< intrrp.C : IntrRp::registerInterrupt */
         MOD_INTR_ENABLE_PSI_INTR     = 0x06, /**< intrrp.C : INTR::enablePsiIntr */
         MOD_INTR_INIT_XIVR           = 0x07, /**< intrrp.C : INTR::initXIVR */
+        MOD_INTR_INIT_MPIPLAREA      = 0x08, /**< intrrp.C : IntrRp::initailizeMpiplSyncArea() */
+        MOD_INTR_SYNC_NODES          = 0x09, /**< intrrp.C : IntrRp::syncNodes() */
+        MOD_INTR_SYNC_ADDNODE        = 0x0A, /**< intrrp.C : IntrRp::addHbNodeToMpiplSyncArea */
+        MOD_INTR_ADDHBNODE           = 0x0B, /**< intrrp.C : INTR::addHbNode */
     };
 
     enum IntrReasonCode
@@ -48,6 +52,7 @@ namespace INTR
         RC_BAD_ISN                    = INTR_COMP_ID | 0x05,
         //termination_rc
         RC_PERSISTENT_INTERRUPTS      = INTR_COMP_ID | 0x06,
+        RC_CANNOT_MAP_MEMORY          = INTR_COMP_ID | 0x07,
     };
 };
 
diff --git a/src/include/usr/vmmconst.h b/src/include/usr/vmmconst.h
index c93e62148..33586a4db 100644
--- a/src/include/usr/vmmconst.h
+++ b/src/include/usr/vmmconst.h
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -158,6 +158,16 @@ enum BlockPriority
                            VMM_ALL_HOMER_OCC_MEMORY_SIZE)
 
 
+/** Internode communication area outside of the HB image.
+ * Preserved between mpipl.
+ *
+ * @node There is one area per hostboot instance.
+ *  Need to add (ATTR_HB_HRMOR_NODAL_BASE * hbinstance_num) to this
+ *  address to get the physical address
+ */
+#define VMM_INTERNODE_PRESERVED_MEMORY_ADDR (96 * MEGABYTE)
+
+
 /**
  * Test Constants
  */
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 1f30832c2..7b3ac859b 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -33,6 +33,7 @@
 #include <util/singleton.H>
 #include <intr/intr_reasoncodes.H>
 #include <sys/mmio.h>
+#include <sys/mm.h>
 #include <sys/misc.h>
 #include <kernel/console.H>
 #include <kernel/ipc.H>
@@ -46,6 +47,7 @@
 #include <sys/vfs.h>
 #include <hwas/common/hwasCallout.H>
 #include <fsi/fsiif.H>
+#include <arch/ppc.H>
 
 #define INTR_TRACE_NAME INTR_COMP_NAME
 
@@ -146,7 +148,6 @@ void IntrRp::init( errlHndl_t   &io_errlHndl_t )
     io_errlHndl_t = err ;
 }
 
-
 //  ICPBAR = INTP.ICP_BAR[0:25] in P8 = 0x3FFFF800 + (8*node) + procPos
 //  P8 Scom address = 0x020109c9
 //
@@ -661,6 +662,17 @@ void IntrRp::msgHandler()
                 }
                 break;
 
+            case MSG_INTR_ADD_HBNODE:  // node info for mpipl
+                {
+                    errlHndl_t err = addHbNodeToMpiplSyncArea(msg->data[0]);
+                    if(err)
+                    {
+                        errlCommit(err,INTR_COMP_ID);
+                    }
+                    msg_free(msg); // async message
+                }
+                break;
+
             default:
                 msg->data[1] = -EINVAL;
                 msg_respond(iv_msgQ, msg);
@@ -2101,6 +2113,12 @@ errlHndl_t IntrRp::hw_disableIntrMpIpl()
             break;
         }
 
+        err = syncNodes(INTR_MPIPL_UPSTREAM_DISABLED);
+        if ( err )
+        {
+            break;
+        }
+
         // Set interrupt presenter to allow all interrupts
         TRACFCOMP(g_trac_intr,"Allow interrupts");
         for(TARGETING::TargetHandleList::iterator
@@ -2127,6 +2145,12 @@ errlHndl_t IntrRp::hw_disableIntrMpIpl()
             break;
         }
 
+        err = syncNodes(INTR_MPIPL_DRAINED);
+        if( err )
+        {
+            break;
+        }
+
         // Disable all interrupt presenters
         for(TARGETING::TargetHandleList::iterator core = procCores.begin();
             core != procCores.end();
@@ -2172,6 +2196,264 @@ errlHndl_t IntrRp::hw_disableIntrMpIpl()
 }
 
 
+errlHndl_t syncNodesError(void * i_p, uint64_t i_len)
+{
+    TRACFCOMP(g_trac_intr,"Failure calling mm_block_map: phys_addr=%p",
+              i_p);
+    /*@
+     * @errortype    ERRL_SEV_UNRECOVERABLE
+     * @moduleid     INTR::MOD_INTR_SYNC_NODES
+     * @reasoncode   INTR::RC_CANNOT_MAP_MEMORY
+     * @userdata1    physical address
+     * @userdata2    Block size requested
+     * @devdesc      Error mapping in memory
+     */
+    return new ERRORLOG::ErrlEntry
+        (
+         ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+         INTR::MOD_INTR_SYNC_NODES,
+         INTR::RC_CANNOT_MAP_MEMORY,
+         reinterpret_cast<uint64_t>(i_p),
+         i_len,
+         true /*Add HB Software Callout*/);
+}
+
+errlHndl_t IntrRp::syncNodes(intr_mpipl_sync_t i_sync_type)
+{
+    errlHndl_t err = NULL;
+    bool reported[MAX_NODES_PER_SYS] = { false,};
+
+    uint64_t hrmorBase = KernelIpc::ipc_data_area.hrmor_base;
+
+    void * node_info_ptr =
+        reinterpret_cast<void *>((iv_masterCpu.nodeId * hrmorBase) +
+                                 VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+    internode_info_t * this_node_info =
+        reinterpret_cast<internode_info_t *>
+        (mm_block_map(node_info_ptr,INTERNODE_INFO_SIZE));
+
+    do
+    {
+
+        if(this_node_info == NULL)
+        {
+            err = syncNodesError(this_node_info, INTERNODE_INFO_SIZE);
+            break;
+        }
+
+
+        if(this_node_info->eye_catcher != NODE_INFO_EYE_CATCHER)
+        {
+            TRACFCOMP(g_trac_intr, INFO_MRK
+                      "MPIPL, but INTR node data sync area unintialized."
+                      " Assuming single HB Intance system");
+
+            break;
+        }
+
+        // Map the internode data areas to a virtual address
+        internode_info_t * vaddr[MAX_NODES_PER_SYS];
+
+        for(uint64_t node = 0; node < MAX_NODES_PER_SYS; ++node)
+        {
+            if (node == iv_masterCpu.nodeId)
+            {
+                vaddr[node] = this_node_info;
+            }
+            else if(this_node_info->exist[node])
+            {
+                node_info_ptr =
+                    reinterpret_cast<void *>
+                    ((node*hrmorBase)+VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+                internode_info_t * node_info =
+                    reinterpret_cast<internode_info_t *>
+                    (mm_block_map(node_info_ptr,
+                                  INTERNODE_INFO_SIZE));
+
+                if(node_info == NULL)
+                {
+                    err = syncNodesError(node_info_ptr,
+                                         INTERNODE_INFO_SIZE);
+                    break;
+                }
+                vaddr[node] = node_info;
+                reported[node] = false;
+            }
+        }
+        if (err)
+        {
+            break;
+        }
+
+
+        // This node has hit the sync point
+        this_node_info->mpipl_intr_sync = i_sync_type;
+        lwsync();
+
+        bool synched = false;
+        // Loop until all nodes have reached the sync point
+        while(synched == false)
+        {
+            synched = true;
+
+            for(uint64_t node = 0; node < MAX_NODES_PER_SYS; ++node)
+            {
+                if(this_node_info->exist[node])
+                {
+                    intr_mpipl_sync_t sync_type =
+                        vaddr[node]->mpipl_intr_sync;
+                    if(sync_type < i_sync_type)
+                    {
+                        synched = false;
+                        // Insure simics does a context switch
+                        setThreadPriorityLow();
+                        setThreadPriorityHigh();
+                    }
+                    else if(reported[node] == false)
+                    {
+                        reported[node] = true;
+                        TRACFCOMP( g_trac_intr, INFO_MRK
+                                   "MPIPL node %ld reached syncpoint %d",
+                                   node, (uint32_t)i_sync_type);
+                    }
+                }
+            }
+        }
+        isync();
+
+        for(uint64_t node = 0; node < MAX_NODES_PER_SYS; ++node)
+        {
+            if(this_node_info->exist[node])
+            {
+                // We are still using this_node_info area
+                // so unmap it later.
+                if(node != iv_masterCpu.nodeId)
+                {
+                    mm_block_unmap(vaddr[node]);
+                }
+            }
+        }
+
+        mm_block_unmap(this_node_info);
+
+    } while(0);
+
+    return err;
+}
+
+
+errlHndl_t  IntrRp::initializeMpiplSyncArea()
+{
+    errlHndl_t err = NULL;
+    uint64_t hrmorBase = KernelIpc::ipc_data_area.hrmor_base;
+    void * node_info_ptr =
+        reinterpret_cast<void *>((iv_masterCpu.nodeId * hrmorBase) +
+                                 VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+    internode_info_t * this_node_info =
+        reinterpret_cast<internode_info_t *>
+        (mm_block_map(node_info_ptr,INTERNODE_INFO_SIZE));
+
+    if(this_node_info)
+    {
+        TRACFCOMP( g_trac_intr,
+                   "MPIPL SYNC at phys %p virt %p value %lx\n",
+                   node_info_ptr, this_node_info, NODE_INFO_EYE_CATCHER );
+
+
+        this_node_info->eye_catcher = NODE_INFO_EYE_CATCHER;
+        this_node_info->version = NODE_INFO_VERSION;
+        this_node_info->mpipl_intr_sync = INTR_MPIPL_SYNC_CLEAR;
+        for(uint64_t node = 0; node < MAX_NODES_PER_SYS; ++node)
+        {
+            if(iv_masterCpu.nodeId == node)
+            {
+                this_node_info->exist[node] = true;
+            }
+            else
+            {
+                this_node_info->exist[node] = false;
+            }
+        }
+
+        mm_block_unmap(this_node_info);
+    }
+    else
+    {
+        TRACFCOMP( g_trac_intr, "Failure calling mm_block_map : phys_addr=%p",
+                   node_info_ptr);
+        /*@
+         * @errortype    ERRL_SEV_UNRECOVERABLE
+         * @moduleid     INTR::MOD_INTR_INIT_MPIPLAREA
+         * @reasoncode   INTR::RC_CANNOT_MAP_MEMORY
+         * @userdata1    physical address
+         * @userdata2    Size
+         * @devdesc      Error mapping in memory
+         */
+        err = new ERRORLOG::ErrlEntry(
+                                      ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                      INTR::MOD_INTR_INIT_MPIPLAREA,
+                                      INTR::RC_CANNOT_MAP_MEMORY,
+                                      reinterpret_cast<uint64_t>(node_info_ptr),
+                                      INTERNODE_INFO_SIZE,
+                                      true /*Add HB Software Callout*/);
+
+    }
+    return err;
+}
+
+errlHndl_t  IntrRp::addHbNodeToMpiplSyncArea(uint64_t i_hbNode)
+{
+    errlHndl_t err = NULL;
+    uint64_t hrmorBase = KernelIpc::ipc_data_area.hrmor_base;
+    void * node_info_ptr =
+        reinterpret_cast<void *>((iv_masterCpu.nodeId * hrmorBase) +
+                                 VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+    internode_info_t * this_node_info =
+        reinterpret_cast<internode_info_t *>
+        (mm_block_map(node_info_ptr,INTERNODE_INFO_SIZE));
+
+    if(this_node_info)
+    {
+        if(this_node_info->eye_catcher != NODE_INFO_EYE_CATCHER)
+        {
+            // Initialize the mutli-node area for this node.
+            err = initializeMpiplSyncArea();
+        }
+
+        this_node_info->exist[i_hbNode] = true;
+        this_node_info->mpipl_intr_sync = INTR_MPIPL_SYNC_CLEAR;
+
+        mm_block_unmap(this_node_info);
+    }
+    else
+    {
+        TRACFCOMP( g_trac_intr, "Failure calling mm_block_map : phys_addr=%p",
+                   node_info_ptr);
+        /*@
+         * @errortype    ERRL_SEV_UNRECOVERABLE
+         * @moduleid     INTR::MOD_INTR_SYNC_ADDNODE
+         * @reasoncode   INTR::RC_CANNOT_MAP_MEMORY
+         * @userdata1    physical address
+         * @userdata2    Size
+         * @devdesc      Error mapping in memory
+         */
+        err = new ERRORLOG::ErrlEntry(
+                                      ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                      INTR::MOD_INTR_SYNC_ADDNODE,
+                                      INTR::RC_CANNOT_MAP_MEMORY,
+                                      reinterpret_cast<uint64_t>(node_info_ptr),
+                                      INTERNODE_INFO_SIZE,
+                                      true /*Add HB Software Callout*/);
+
+    }
+    return err;
+}
+
+
 //----------------------------------------------------------------------------
 // External interfaces
 //----------------------------------------------------------------------------
@@ -2433,3 +2715,41 @@ void* INTR::IntrRp::handleCpuTimeout(void* _pir)
 
     return NULL;
 }
+
+errlHndl_t INTR::addHbNode(uint64_t i_hbNode)
+{
+    errlHndl_t err = NULL;
+    msg_q_t intr_msgQ = msg_q_resolve(VFS_ROOT_MSG_INTR);
+    TRACFCOMP( g_trac_intr,"Add node %d for MPIPL",i_hbNode);
+    if(intr_msgQ)
+    {
+        msg_t * msg = msg_allocate();
+        msg->data[0] = i_hbNode;
+        msg->type = MSG_INTR_ADD_HBNODE;
+        msg_send(intr_msgQ, msg);
+    }
+    else
+    {
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_INFORMATIONAL
+         * @moduleid        INTR::MOD_INTR_ADDHBNODE
+         * @reasoncode      INTR::RC_RP_NOT_INITIALIZED
+         * @userdata1       MSG_INTR_ADD_HBNODE
+         * @userdata2       hbNode to add
+         *
+         * @devdesc         Interrupt resource provider not initialized yet.
+         *
+         */
+        err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_INFORMATIONAL,      // severity
+             INTR::MOD_INTR_ADDHBNODE,              // moduleid
+             INTR::RC_RP_NOT_INITIALIZED,           // reason code
+             static_cast<uint64_t>(MSG_INTR_ADD_HBNODE),
+             i_hbNode
+            );
+    }
+
+    return err;
+}
+
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 69c4be68b..b32cf5738 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -31,6 +31,7 @@
 #include <sys/msg.h>
 #include <sys/misc.h>
 #include <sys/time.h>
+#include <sys/internode.h>
 #include <intr/interrupt.H>
 #include <map>
 #include <algorithm>
@@ -539,6 +540,28 @@ namespace INTR
             void shutDown();
 
             /**
+             * Wait for all nodes to disable interrupts on MPIPL
+             * @param[in] i_sync_type, the type of event to sync on
+             *            @see src/include/sys/internode.h
+             * @return err log handle
+             */
+            errlHndl_t syncNodes(intr_mpipl_sync_t i_sync_type);
+
+            /**
+             * Initailize the mpipl node sync data area for this HB instance
+             * @return err log handle
+             */
+            errlHndl_t  initializeMpiplSyncArea();
+
+            /**
+             * Add the existance of another HB instance to this instance
+             * mpipl sync data area.
+             * @param[in] The instance number of the HB instance to add
+             * @return error log handle
+             */
+            errlHndl_t  addHbNodeToMpiplSyncArea(uint64_t i_hbNode);
+
+            /**
              * Calculate the adress offset for the given cpu
              * @param[in] i_pir PIR value for the presenter
              * @return the offset
diff --git a/src/usr/intr/test/intrtest.H b/src/usr/intr/test/intrtest.H
index c653d6609..bbc8b07a3 100644
--- a/src/usr/intr/test/intrtest.H
+++ b/src/usr/intr/test/intrtest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -34,6 +34,8 @@
 #include <sys/mmio.h>
 #include <sys/task.h>
 #include <sys/misc.h>
+#include <sys/internode.h>
+#include <sys/mm.h>
 #include <kernel/intmsghandler.H>
 
 extern trace_desc_t * g_trac_intr;
@@ -156,6 +158,72 @@ class IntrTest: public CxxTest::TestSuite
             }
         }
 
+        void test_mpipl_node_data( void )
+        {
+            errlHndl_t err = NULL;
+            for(uint64_t hb_node = 0; hb_node < MAX_NODES_PER_SYS; ++hb_node)
+            {
+                // This message is async so the NODE_INFO_AREA may take
+                // a bit to show the changes
+                err = INTR::addHbNode(hb_node);
+
+                if ( err )
+                {
+                    break;
+                }
+            }
+
+            if ( err )
+            {
+                TS_FAIL("Error log from INTR::addHbNode");
+                errlCommit(err, INTR_COMP_ID);
+            }
+
+            // Unregister a non-existant mesgq - This not only tests this
+            // interface using a bad param, but also serves to synchonize the
+            // async messages above so we don't check the NODE_DATA_AREA before
+            // it has been set.
+            msg_q_t msgQ = INTR::unRegisterMsgQ(0xff);
+            if(msgQ != NULL)
+            {
+                TS_FAIL("INTR: non-existant external interrupt type returned "
+                        "a valid msgQ");
+            }
+
+            void * node_info_ptr =
+                reinterpret_cast<void *>(VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+            internode_info_t * this_node_info =
+                reinterpret_cast<internode_info_t *>
+                (mm_block_map(node_info_ptr,INTERNODE_INFO_SIZE));
+
+            if(this_node_info)
+            {
+                if(this_node_info->eye_catcher != NODE_INFO_EYE_CATCHER)
+                {
+                    TS_FAIL("INTR: NODE_DATA_AREA not initialized");
+                }
+                for(uint64_t hb_node = 0;
+                    hb_node < MAX_NODES_PER_SYS;
+                    ++hb_node)
+                {
+                    if(this_node_info->exist[hb_node] != true)
+                    {
+                        TS_FAIL
+                            ("INTR: NODE_DATA_AREA.exist not true for node %d",
+                             hb_node);
+                    }
+                }
+
+                mm_block_unmap(this_node_info);
+            }
+            else
+            {
+                TS_FAIL("INTR Could not map memory of NODE_DATA_AREA");
+            }
+
+        }
+
 
         IntrTest() : CxxTest::TestSuite()
         {
diff --git a/src/usr/mbox/ipcSp.C b/src/usr/mbox/ipcSp.C
index ac45be232..db4dc9b42 100644
--- a/src/usr/mbox/ipcSp.C
+++ b/src/usr/mbox/ipcSp.C
@@ -28,6 +28,7 @@
 #include <mbox/mboxif.H>
 #include <errl/errlmanager.H>
 #include <mbox/mbox_reasoncodes.H>
+#include <intr/interrupt.H>
 
 namespace START_PAYLOAD
 {
@@ -142,9 +143,17 @@ void IpcSp::msgHandler()
                            "IPC received the test connection msg - %d:%d",
                             msg->data[0], msg->data[1] );
 
+                // Tell this HB node about the other HB node
+                err = INTR::addHbNode(msg->data[1]);
+                if( err)
+                {
+                    errlCommit(err,IPC_COMP_ID);
+                }
+
                 //Send a response to indicate the connection has been
                 //established
                 err = MBOX::send(MBOX::HB_COALESCE_MSGQ, msg, msg->data[1] );
+
                 if (err)
                 {
                     errlCommit(err,IPC_COMP_ID);
diff --git a/src/usr/mbox/mailboxsp.C b/src/usr/mbox/mailboxsp.C
index 59b5dff04..28b6f4346 100644
--- a/src/usr/mbox/mailboxsp.C
+++ b/src/usr/mbox/mailboxsp.C
@@ -1665,6 +1665,7 @@ errlHndl_t MBOX::send(queue_id_t i_q_id, msg_t * i_msg,int i_node)
         {
             uint64_t q_handle = i_q_id;
             q_handle |= (((uint64_t)MSGQ_TYPE_IPC | (uint64_t)i_node) << 32);
+
             TRACFCOMP(g_trac_mboxmsg,INFO_MRK
                       "MBOXSP IPC SEND MSG: Dest node %d. msg_id: %lx",
                       i_node,