Add error callouts and other improvements for Node Communications

This code adds bus, hw, code, etc callouts to the appropriate errors.
It also cleans up other TODOs, refactors some functions, and applies
somes fixes found in testing.

Change-Id: I462cdb2c4e313c277a3e0542f740e82ede3313f3
RTC:184518
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/61813
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

5 files changed, 610 insertions, 331 deletions

diff --git a/src/usr/secureboot/node_comm/node_comm.C b/src/usr/secureboot/node_comm/node_comm.C
index cdaf51755..95008344a 100644
--- a/src/usr/secureboot/node_comm/node_comm.C
+++ b/src/usr/secureboot/node_comm/node_comm.C
@@ -33,6 +33,8 @@
 // Includes
 // ----------------------------------------------
 #include <string.h>
+#include <sys/time.h>
+#include <sys/task.h>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <errl/errludtarget.H>
@@ -55,14 +57,220 @@ namespace SECUREBOOT
 namespace NODECOMM
 {
 
+// ----------------------------------------------
+// Defines
+// ----------------------------------------------
+// If the link(s) are up the operation should complete right away
+// so there will only be a short polling window
+#define NODE_COMM_POLL_DELAY_NS (10 * NS_PER_MSEC)  // Sleep for 10ms per poll
+// FSP is expecting a reply in 30 seconds, so leave some buffer
+#define NODE_COMM_POLL_DELAY_TOTAL_NS (25 * NS_PER_SEC) // Total time 25s
+
+
+/**
+ *  @brief This function waits for the processor to receive a message over
+ *         ABUS from a processor on another node.
+ */
+errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
+                                   uint64_t & o_data,
+                                   uint8_t & o_linkId,
+                                   uint8_t & o_mboxId)
+{
+    errlHndl_t err = nullptr;
+    bool attn_found = false;
+
+    const uint64_t interval_ns = NODE_COMM_POLL_DELAY_NS;
+    uint64_t time_polled_ns = 0;
+
+    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: pProc=0x%.08X",
+              get_huid(i_pProc));
+
+    do
+    {
+        do
+        {
+
+        // Look for Attention
+        err = nodeCommMapAttn(i_pProc,
+                              NCDD_MODE_ABUS,
+                              attn_found,
+                              o_linkId,
+                              o_mboxId);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: Error Back "
+                      "From nodeCommMapAttn: Tgt=0x%.08X: "
+                      TRACE_ERR_FMT,
+                      get_huid(i_pProc),
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+        if (attn_found == true)
+        {
+            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
+              "nodeCommMapAttn attn_found (%d) for Tgt=0x%.08X, link=%d, "
+              "mbox=%d",
+              attn_found, get_huid(i_pProc), o_linkId, o_mboxId);
+            break;
+        }
+
+        if (time_polled_ns >= NODE_COMM_POLL_DELAY_TOTAL_NS)
+        {
+            TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusRecvMessage: "
+              "timeout: time_polled_ns-0x%.16llX, MAX=0x%.16llX, "
+              "interval=0x%.16llX",
+              time_polled_ns, NODE_COMM_POLL_DELAY_TOTAL_NS, interval_ns);
+
+            /*@
+             * @errortype
+             * @reasoncode       RC_NC_WAITING_TIMEOUT
+             * @moduleid         MOD_NC_RECV
+             * @userdata1[0:31]  Master Proc Target HUID
+             * @userdata1[32:63] Time Polled in ns
+             * @userdata2[0:31]  Defined MAX Poll Time in ns
+             * @userdata2[32:63] Time Interval Between Polls in ns
+             * @devdesc          Timed out waiting to receive message over
+             *                   ABUS Link Mailbox
+             * @custdesc         Secure Boot failure
+             */
+            err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                           MOD_NC_RECV,
+                                           RC_NC_WAITING_TIMEOUT,
+                                           TWO_UINT32_TO_UINT64(
+                                             get_huid(i_pProc),
+                                             time_polled_ns),
+                                           TWO_UINT32_TO_UINT64(
+                                             NODE_COMM_POLL_DELAY_TOTAL_NS,
+                                             interval_ns));
+
+            // Bus Callout will be handled by caller since
+            // it should know expected peer target
+
+            // Or HB code failed to do the procedure correctly
+            err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                     HWAS::SRCI_PRIORITY_LOW);
+
+            // Grab FFDC from the target
+            getNodeCommFFDC(NCDD_MODE_ABUS,
+                            i_pProc,
+                            err);
+
+            break;
+        }
+
+        // Sleep before polling again
+        nanosleep( 0, interval_ns );
+        task_yield(); // wait patiently
+        time_polled_ns += interval_ns;
+
+        } while(attn_found == false);
+
+    if (err)
+    {
+        break;
+    }
+
+    if (attn_found == true)
+    {
+        //  Read message on proc with Link Mailbox found above
+        o_data = 0;
+        size_t expSize = sizeof(o_data);
+        auto reqSize = expSize;
+        err = DeviceFW::deviceRead(i_pProc,
+                                   &o_data,
+                                   reqSize,
+                                   DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
+                                                           o_linkId,
+                                                           o_mboxId));
+
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
+                      "Abus MBox Read: Tgt=0x%.08X, link=%d, mbox=%d: "
+                      TRACE_ERR_FMT,
+                      get_huid(i_pProc), o_linkId, o_mboxId,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+        assert(reqSize==expSize,"nodeCommRecvMessage: SCOM deviceRead didn't return expected data size of %d (it was %d)",
+               expSize,reqSize);
+
+    }
+
+    } while( 0 );
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusRecvMessage: "
+              "Tgt=0x%.08X, link=%d, mbox=%d attn_found=%d: "
+              "data=0x%.16llX. "
+              TRACE_ERR_FMT,
+              get_huid(i_pProc), o_linkId, o_mboxId, attn_found, o_data,
+              TRACE_ERR_ARGS(err));
+
+    return err;
+
+} // end of nodeCommAbusRecvMessage
+
+
+/**
+ *  @brief This function sends a message over the ABUS from the processor of
+ *         the current node to a processor on another node.
+ */
+errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
+                                   const uint64_t & i_data,
+                                   const uint8_t & i_linkId,
+                                   const uint8_t & i_mboxId)
+{
+    errlHndl_t err = nullptr;
+
+    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
+              "to send data=0x%.16llX through linkId=%d mboxId=%d",
+              get_huid(i_pProc), i_data, i_linkId, i_mboxId);
+
+    do
+    {
+        // Send Data
+        uint64_t data = i_data; // to keep i_data const
+        size_t expSize = sizeof(i_data);
+        auto reqSize = expSize;
+        err = DeviceFW::deviceWrite(i_pProc,
+                                    &data,
+                                    reqSize,
+                                    DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
+                                                            i_linkId,
+                                                            i_mboxId));
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusSendMessage: Error Back "
+                      "From Abus MBox Send: Tgt=0x%.08X, data=0x%.16llX, "
+                      "link=%d, mbox=%d: "
+                      TRACE_ERR_FMT,
+                      get_huid(i_pProc), i_data, i_linkId, i_mboxId,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+        assert(reqSize==expSize,"nodeCommAbusSendMessage: SCOM deviceRead didn't return expected data size of %d (it was %d)",
+               expSize,reqSize);
+
+    } while( 0 );
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: "
+              TRACE_ERR_FMT,
+              TRACE_ERR_ARGS(err));
+
+    return err;
+
+} // end of nodeCommAbusSendMessage
+
+
+
 /**
  *  @brief Map Attention Bits in FIR Register to specific Link Mailbox
  */
 errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
                            const node_comm_modes_t i_mode,
                            bool & o_attn_found,
-                           uint64_t & o_linkId,
-                           uint64_t & o_mboxId)
+                           uint8_t & o_linkId,
+                           uint8_t & o_mboxId)
 {
     errlHndl_t err = nullptr;
     uint64_t fir_data = 0x0;
@@ -270,6 +478,166 @@ void getNodeCommFFDC( node_comm_modes_t   i_mode,
 } // end of getNodeCommFFDC
 
 
+/**
+ * @brief Add a bus callout to an error log
+ */
+void addNodeCommBusCallout(node_comm_modes_t i_mode,
+                           TARGETING::Target* i_pProc,
+                           uint8_t & i_linkId,
+                           errlHndl_t & io_log,
+                           HWAS::callOutPriority i_priority)
+{
+    TRACFCOMP(g_trac_nc,ENTER_MRK
+              "addNodeCommBusCallout: tgt=0x%X, mode=%s, linkId=%d, "
+              "err_plid=0x%X, priority=%d",
+              get_huid(i_pProc),
+              (i_mode == NCDD_MODE_ABUS)
+                ? NCDD_ABUS_STRING : NCDD_XBUS_STRING,
+              i_linkId,
+              ERRL_GETPLID_SAFE(io_log), i_priority);
+
+    // Bus associated with i_pProc and i_linkId (PHYS_PATH)
+    char * l_ep1_path_str = nullptr;
+
+    // PEER_PATH associated with l_ep1
+    char * l_ep2_path_str = nullptr;
+
+    bool found_peer_endpoint = false;
+
+    do
+    {
+    if ((io_log == nullptr) ||
+        (i_pProc == nullptr) ||
+        ((i_mode != NCDD_MODE_ABUS) &&
+         (i_mode != NCDD_MODE_XBUS)))
+    {
+        TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: io_log==nullptr or "
+                  "tgt=0x%X is nullptr or i_mode=%d is invalid (not ABUS (%d) "
+                  "or XBUS (%d)) so no bus callout has beed added",
+                  get_huid(i_pProc), i_mode, NCDD_MODE_ABUS, NCDD_MODE_XBUS);
+        break;
+    }
+
+    // Get Bus Type
+    HWAS::busTypeEnum l_bus_type = (i_mode == NCDD_MODE_ABUS)
+                                    ? HWAS::O_BUS_TYPE : HWAS::X_BUS_TYPE; // using O_ instead of A_BUS_TYPE
+
+    // Get all Chiplets for this target aligned with the input mode
+    TargetHandleList l_busTargetList;
+    TYPE l_type = (i_mode == NCDD_MODE_ABUS)
+                   ? TYPE_OBUS : TYPE_XBUS;
+    getChildChiplets(l_busTargetList, i_pProc, l_type);
+
+    // Get BUS Instance
+    // For each OBUS or XBUS instance there are 2 links and 2 mailboxes,
+    // so divide the linkId by 2 to get bus instance
+    uint8_t l_bus_instance_ep1 = i_linkId / 2;
+
+
+    // Look through Bus Targets looking for specific Bus Instance
+    // to find the right PEER_TARGET
+    for (const auto & l_busTgt : l_busTargetList)
+    {
+        found_peer_endpoint = false;
+        EntityPath l_ep1 = l_busTgt->getAttr<ATTR_PHYS_PATH>();
+        if (l_ep1_path_str != nullptr)
+        {
+            free(l_ep1_path_str);
+        }
+        l_ep1_path_str = l_ep1.toString();
+
+
+        EntityPath l_ep2 = l_busTgt->getAttr<ATTR_PEER_PATH>();
+        if (l_ep2_path_str != nullptr)
+        {
+            free(l_ep2_path_str);
+        }
+        l_ep2_path_str = l_ep2.toString();
+
+        TRACUCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: Checking "
+                      "i_pProc 0x%.08X BUS HUID 0x%.08X's (%s) PEER_PATH: %s",
+                      get_huid(i_pProc), get_huid(l_busTgt),
+                      l_ep1_path_str,
+                      l_ep2_path_str);
+
+        EntityPath::PathElement l_ep2_peBus =
+                                  l_ep2.pathElementOfType(l_type);
+        if(l_ep2_peBus.type == TYPE_NA)
+        {
+            TRACUCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                      "Skipping i_pProc 0x%.08X "
+                      "BUS HUID 0x%.08X's (%s) PEER_PATH %s because "
+                      "cannot find BUS in PEER_PATH",
+                      get_huid(i_pProc), get_huid(l_busTgt),
+                      l_ep1_path_str, l_ep2_path_str);
+            continue;
+        }
+
+        if (l_bus_instance_ep1 == l_busTgt->getAttr<ATTR_REL_POS>())
+        {
+            TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                      "Using i_pProc 0x%.08X BUS HUID 0x%.08X (%s) "
+                      "PEER_PATH %s as it had right instance %d for ep1",
+                      get_huid(i_pProc), get_huid(l_busTgt), l_ep1_path_str,
+                      l_ep2_path_str, l_bus_instance_ep1);
+
+            found_peer_endpoint = true;
+
+            // Add Bus Callout
+            io_log->addBusCallout(l_ep1,
+                                  l_ep2,
+                                  l_bus_type,
+                                  i_priority);
+
+           // @TODO RTC 184518 - Possibly go to depth of TYPE_SMPGROUP,
+           // which is one level below OBUS for ABUS/OBUS callouts
+
+            break;
+        }
+        else
+        {
+            TRACUCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                      "Skipping i_pProc 0x%.08X BUS HUID 0x%.08X's "
+                      "PEER_PATH %s because ep1 bus instance (%d) does not "
+                      "match instance (%d) converted from i_linkId (%d)",
+                      get_huid(i_pProc), get_huid(l_busTgt),
+                      l_ep2_path_str, l_busTgt->getAttr<ATTR_ORDINAL_ID>(),
+                      l_bus_instance_ep1, i_linkId);
+        }
+    }
+
+    if (found_peer_endpoint == false)
+    {
+        TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: Unable to find a "
+                  "peer_tgt for tgt=0x%X, mode=%s, linkId=%d, so no bus "
+                  "callout has been added",
+                  get_huid(i_pProc),
+                  (i_mode == NCDD_MODE_ABUS)
+                    ? NCDD_ABUS_STRING : NCDD_XBUS_STRING,
+                  i_linkId);
+        break;
+    }
+
+    } while( 0 );
+
+    if (l_ep1_path_str != nullptr)
+    {
+        free(l_ep1_path_str);
+        l_ep1_path_str = nullptr;
+    }
+
+    if (l_ep2_path_str != nullptr)
+    {
+        free(l_ep2_path_str);
+        l_ep2_path_str = nullptr;
+    }
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"addNodeCommBusCallout");
+
+    return;
+
+} // end of addNodeCommBusCallout
+
 } // End NODECOMM namespace
 
 } // End SECUREBOOT namespace
diff --git a/src/usr/secureboot/node_comm/node_comm.H b/src/usr/secureboot/node_comm/node_comm.H
index 5138e91d0..7ed48ffc9 100644
--- a/src/usr/secureboot/node_comm/node_comm.H
+++ b/src/usr/secureboot/node_comm/node_comm.H
@@ -29,6 +29,7 @@
 // Includes
 // ----------------------------------------------
 #include <config.h>
+#include <time.h>
 #include <devicefw/userif.H>
 #include <trace/interface.H>
 #include <scom/centaurScomCache.H> // for TRACE_ERR_FMT, TRACE_ERR_ARGS
@@ -176,6 +177,45 @@ inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
 
 
 /**
+ *  @brief This function waits for the processor to receive a message over
+ *         ABUS from a processor on another node.
+ *
+ *  @param[in]  i_pProc  - Processor target to look for attentions on
+ *                         Can't be nullptr
+ *  @param[out] o_data   - Data received
+ *  @param[out] o_linkId - Link Id that received the message
+ *  @param[out] o_mboxId - Mailbox Id that received the message
+ *
+ *  @return errlHndl_t Error log handle
+ *  @retval nullptr Operation was successful
+ *  @retval !nullptr Operation failed with valid error log
+ */
+errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
+                                   uint64_t & o_data,
+                                   uint8_t & o_linkId,
+                                   uint8_t & o_mboxId);
+
+/**
+ *  @brief This function sends a message over the ABUS from the processor of
+ *         the current node to a processor on another node.
+ *
+ *  @param[in] i_pProc  - Processor target to look for attentions on
+ *                        Can't be nullptr
+ *  @param[in] i_data   - Data to be sent
+ *  @param[in] i_linkId - Link Id Message is sent from
+ *  @param[in] i_mboxId - Mailbox Id Message is sent from
+ *
+ *  @return errlHndl_t Error log handle
+ *  @retval nullptr Operation was successful
+ *  @retval !nullptr Operation failed with valid error log
+ */
+errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
+                                   const uint64_t & i_data,
+                                   const uint8_t & i_linkId,
+                                   const uint8_t & i_mboxId);
+
+
+/**
  *  @brief Map Attention Bits in XBUS/ABUS FIR Register to specific Link Mailbox
  *
  *  @param[in] i_pProc      Processor target to look for attentions on
@@ -193,8 +233,8 @@ inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
 errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
                            node_comm_modes_t i_mode,
                            bool & o_attn_found,
-                           uint64_t & o_linkId,
-                           uint64_t & o_mboxId);
+                           uint8_t & o_linkId,
+                           uint8_t & o_mboxId);
 
 /**
  * @brief Add FFDC for the target to an error log
@@ -206,9 +246,29 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
  * @return void
  */
 void getNodeCommFFDC(node_comm_modes_t i_mode,
-                     TARGETING::Target*  i_pProc,
+                     TARGETING::Target* i_pProc,
                      errlHndl_t & io_log);
 
+/**
+ * @brief Adds a bus callout to the error log by determining the bus and the two
+ *        targets associated with it
+ *
+ * @param[in] i_mode       Specifies XBUS or ABUS mode
+ * @param[in] i_pProc      Proc Chip Target used to collect info from
+ *                         Can't be nullptr
+ * @param[in] i_linkId     Link ID associated with i_pProc
+ * @param[in,out] io_log   Error Log to add FFDC to
+ * @param[in] i_priority   Optional: Callout Priority (defaulted to HIGH)
+ *
+ * @return void
+ */
+void addNodeCommBusCallout(
+    node_comm_modes_t i_mode,
+    TARGETING::Target* i_pProc,
+    uint8_t & i_linkId,
+    errlHndl_t & io_log,
+    HWAS::callOutPriority i_priority = HWAS::SRCI_PRIORITY_HIGH);
+
 } // end NODECOMM namespace
 
 } // end SECUREBOOT namespace
diff --git a/src/usr/secureboot/node_comm/node_comm_dd.C b/src/usr/secureboot/node_comm/node_comm_dd.C
index a94454507..dd494ebaf 100644
--- a/src/usr/secureboot/node_comm/node_comm_dd.C
+++ b/src/usr/secureboot/node_comm/node_comm_dd.C
@@ -418,16 +418,14 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
                                        i_args.tgt_huid);
 
         // Likely an issue with Processor or its bus
-        err->addHwCallout( i_args.tgt,
-                           HWAS::SRCI_PRIORITY_HIGH,
-                           HWAS::DELAYED_DECONFIG,
-                           HWAS::GARD_NULL );
+        addNodeCommBusCallout(i_args.mode,
+                              i_args.tgt,
+                              i_args.linkId,
+                              err);
 
         // Or HB code failed to do the procedure correctly
         err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                 HWAS::SRCI_PRIORITY_MED);
-
-        // @TODO RTC 184518 - Look into bus callouts
+                                 HWAS::SRCI_PRIORITY_LOW);
 
         break;
     }
@@ -512,18 +510,16 @@ errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
                                            i_args.tgt_huid);
 
             // Likely an issue with Processor or its bus
-            err->addHwCallout( i_args.tgt,
-                               HWAS::SRCI_PRIORITY_HIGH,
-                               HWAS::DELAYED_DECONFIG,
-                               HWAS::GARD_NULL );
+            addNodeCommBusCallout(i_args.mode,
+                                  i_args.tgt,
+                                  i_args.linkId,
+                                  err);
 
             // Or HB code failed to do the procedure correctly
             err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_MED);
-
-            // @TODO RTC 184518 - Look into bus callouts
+                                     HWAS::SRCI_PRIORITY_LOW);
 
-           break;
+            break;
         }
 
 
@@ -612,16 +608,14 @@ errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args,
                                                 i_args.tgt_huid));
 
                 // Likely an issue with Processor or its bus
-                err->addHwCallout(i_args.tgt,
-                                  HWAS::SRCI_PRIORITY_HIGH,
-                                  HWAS::DELAYED_DECONFIG,
-                                  HWAS::GARD_NULL);
+                addNodeCommBusCallout(i_args.mode,
+                                      i_args.tgt,
+                                      i_args.linkId,
+                                      err);
 
                 // Or HB code failed to do the procedure correctly
                 err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                         HWAS::SRCI_PRIORITY_MED);
-
-                // @TODO RTC 184518 - Look into bus callouts
+                                         HWAS::SRCI_PRIORITY_LOW);
 
                 break;
             }
diff --git a/src/usr/secureboot/node_comm/node_comm_exchange.C b/src/usr/secureboot/node_comm/node_comm_exchange.C
index 8acf7ee70..49fd5a36b 100644
--- a/src/usr/secureboot/node_comm/node_comm_exchange.C
+++ b/src/usr/secureboot/node_comm/node_comm_exchange.C
@@ -50,8 +50,6 @@
 #include <targeting/common/utilFilter.H>
 #include <targeting/targplatutil.H>
 #include <sys/internode.h>
-#include <sys/time.h>
-#include <sys/task.h>
 #include <util/misc.H>
 #include <config.h>
 
@@ -60,12 +58,6 @@
 // ----------------------------------------------
 // Defines
 // ----------------------------------------------
-// If the link(s) are up the operation should complete right away
-// so there will only be a short polling window
-#define NODE_COMM_POLL_DELAY_NS (10 * NS_PER_MSEC)  // Sleep for 10ms per poll
-// FSP is expecting a reply in 30 seconds, so leave some buffer
-#define NODE_COMM_POLL_DELAY_TOTAL_NS (25 * NS_PER_SEC) // Total time 25s
-
 // Use if there is an issue getting random number from a functional TPM
 #define NODE_COMM_DEFAULT_NONCE 0xFFFFFFFFFFFFFFFF
 
@@ -209,7 +201,7 @@ errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
     }
 
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusGetRandom: "
-              "nonce=0x%.16llX, from TPM=0x%.08X: "
+              "nonce=0x%.16llX from TPM=0x%.08X. "
               TRACE_ERR_FMT,
               o_nonce, get_huid(tpm_tgt),
               TRACE_ERR_ARGS(err));
@@ -249,7 +241,7 @@ errlHndl_t nodeCommAbusLogNonce(uint64_t & i_nonce)
                                  "Node Nonce");
     if (err)
     {
-       TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusGetRandom: pcrExtend "
+       TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusLogNonce: pcrExtend "
                   "returned a fail: "
                   TRACE_ERR_FMT,
                   TRACE_ERR_ARGS(err));
@@ -268,267 +260,6 @@ errlHndl_t nodeCommAbusLogNonce(uint64_t & i_nonce)
 
 
 /**
- *  @brief This function waits for the master processor of the current node to
- *         receive a message over ABUS from a master processor on another node.
- *         This function also handles storing the message received into the TPM.
- *
- *  @param[in]  i_mProcInfo - Information about Master Proc
- *  @param[out] o_linkId - Link Id that received the message
- *  @param[out] o_mboxId - Mailbox Id that received the message
- *
- *  @return errlHndl_t Error log handle
- *  @retval nullptr Operation was successful
- *  @retval !nullptr Operation failed with valid error log
- */
-errlHndl_t nodeCommAbusRecvMessage(const master_proc_info_t & i_mProcInfo,
-                                   uint64_t & o_linkId,
-                                   uint64_t & o_mboxId)
-{
-
-    errlHndl_t err = nullptr;
-    bool attn_found = false;
-
-    const uint64_t interval_ns = NODE_COMM_POLL_DELAY_NS;
-    uint64_t time_polled_ns = 0;
-
-    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: mProc=0x%.08X",
-              get_huid(i_mProcInfo.tgt));
-
-    do
-    {
-        do
-        {
-
-        // Look for Attention
-        err = nodeCommMapAttn(i_mProcInfo.tgt,
-                              NCDD_MODE_ABUS,
-                              attn_found,
-                              o_linkId,
-                              o_mboxId);
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: Error Back "
-                      "From nodeCommMapAttn: Tgt=0x%.08X: "
-                      TRACE_ERR_FMT,
-                      get_huid(i_mProcInfo.tgt),
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-        if (attn_found == true)
-        {
-            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
-              "nodeCommMapAttn attn_found (%d) for Tgt=0x%.08X, link=%d, "
-              "mbox=%d",
-              attn_found, get_huid(i_mProcInfo.tgt), o_linkId, o_mboxId);
-            break;
-        }
-
-        if (time_polled_ns >= NODE_COMM_POLL_DELAY_TOTAL_NS)
-        {
-            TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusRecvMessage: "
-              "timeout: time_polled_ns-0x%.16llX, MAX=0x%.16llX, "
-              "interval=0x%.16llX",
-              time_polled_ns, NODE_COMM_POLL_DELAY_TOTAL_NS, interval_ns);
-
-            /*@
-             * @errortype
-             * @reasoncode       RC_NCEX_WAITING_TIMEOUT
-             * @moduleid         MOD_NCEX_RECV
-             * @userdata1[0:31]  Master Proc Target HUID
-             * @userdata1[32:63] Time Polled in ns
-             * @userdata2[0:31]  Defined MAX Poll Time in ns
-             * @userdata2[32:63] Time Interval Between Polls in ns
-             * @devdesc          Timed out waiting to receive message over
-             *                   ABUS Link Mailbox
-             * @custdesc         Secure Boot failure
-             */
-            err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                           MOD_NCEX_RECV,
-                                           RC_NCEX_WAITING_TIMEOUT,
-                                           TWO_UINT32_TO_UINT64(
-                                             get_huid(i_mProcInfo.tgt),
-                                             time_polled_ns),
-                                           TWO_UINT32_TO_UINT64(
-                                             NODE_COMM_POLL_DELAY_TOTAL_NS,
-                                             interval_ns));
-
-            // Likely an issue with Processor or its bus
-            err->addHwCallout( i_mProcInfo.tgt,
-                               HWAS::SRCI_PRIORITY_HIGH,
-                               HWAS::DELAYED_DECONFIG,
-                               HWAS::GARD_NULL );
-
-            // Or HB code failed to do the procedure correctly
-            err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_MED);
-
-            // @TODO RTC 184518 - Add bus callout
-            break;
-        }
-
-        // Sleep before polling again
-        nanosleep( 0, interval_ns );
-        task_yield(); // wait patiently
-        time_polled_ns += interval_ns;
-
-        } while(attn_found == false);
-
-    if (err)
-    {
-        break;
-    }
-
-    if (attn_found == true)
-    {
-        //  Read message on proc with Link Mailbox found above
-        uint64_t data = 0;
-        size_t size = sizeof(data);
-        err = DeviceFW::deviceRead(i_mProcInfo.tgt,
-                                   &data,
-                                   size,
-                                   DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
-                                                           o_linkId,
-                                                           o_mboxId));
-
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
-                      "Abus MBox Read: Tgt=0x%.08X, link=%d, mbox=%d: "
-                      TRACE_ERR_FMT,
-                      get_huid(i_mProcInfo.tgt), o_linkId, o_mboxId,
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-        // Add receiver Link Id to the message data
-        msg_format_t msg_data;
-        msg_data.value = data;
-        msg_data.receiver_linkId = o_linkId;
-        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommRecvMessage: Msg retrieved = "
-                  "0x%.16llX. After adding recv Link Id, 0x%.16llX will be "
-                  "stored in the TPM",
-                  data, msg_data.value);
-
-
-        // Push this msg_data to TPM
-        err = nodeCommAbusLogNonce(msg_data.value);
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
-                      "nodeCommAbusLogNonce: "
-                      TRACE_ERR_FMT,
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-
-    }
-
-    } while( 0 );
-
-    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusRecvMessage: "
-              "Tgt=0x%.08X, link=%d, mbox=%d attn_found=%d: "
-              TRACE_ERR_FMT,
-              get_huid(i_mProcInfo.tgt), o_linkId, o_mboxId, attn_found,
-              TRACE_ERR_ARGS(err));
-
-    return err;
-
-} // end of nodeCommAbusRecvMessage
-
-/**
- *  @brief This function sends a message over the ABUS from the
- *          master processor of the current node to a master processor
- *          on another node. This function also handles the generation
- *          of a random number from the TPM to be used as part of the message.
- *
- *  @param[in] i_mProcInfo - Information about Master Proc
- *  @param[in] i_send_linkId - Link Id Message is sent from
- *  @param[in] i_send_mboxId - Mailbox Id Message is sent from
- *  @param[in] i_peer_linkId - Link Id Message is expected to be received on
- *                             NOTE: this parameter is used for nonce logging/
- *                             extending purposes only
- *
- *  @return errlHndl_t Error log handle
- *  @retval nullptr Operation was successful
- *  @retval !nullptr Operation failed with valid error log
- */
-errlHndl_t nodeCommAbusSendMessage(const master_proc_info_t & i_mProcInfo,
-                                   const uint8_t & i_send_linkId,
-                                   const uint8_t & i_send_mboxId,
-                                   const uint8_t & i_peer_linkId)
-{
-
-    errlHndl_t err = nullptr;
-
-    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: mProc=0x%.08X "
-              "to communicate through linkId=%d mboxId=%d to peer linkId=%d",
-              get_huid(i_mProcInfo.tgt), i_send_linkId,
-              i_send_mboxId, i_peer_linkId);
-
-    do
-    {
-        // Get random number from TPM
-        msg_format_t msg_data;
-        msg_data.value = 0;
-
-        err = nodeCommAbusGetRandom(msg_data.value);
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusSendMessage: Error Back "
-                      "From nodeCommAbusGetRandom: "
-                      TRACE_ERR_FMT,
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-        // Set the send and expected receive LinkIds in the nonce
-        msg_data.origin_linkId = i_send_linkId;
-        msg_data.receiver_linkId = i_peer_linkId;
-
-        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusSendMessage: "
-                  "send linkId=%d, mboxId=%d; peer/recv linkId=%d: "
-                  "data=0x%.016llX",
-                  i_send_linkId, i_send_mboxId, i_peer_linkId, msg_data.value);
-
-        // Send Data
-        size_t size = sizeof(msg_data.value);
-        err = DeviceFW::deviceWrite(i_mProcInfo.tgt,
-                                    &msg_data.value,
-                                    size,
-                                    DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
-                                                            i_send_linkId,
-                                                            i_send_mboxId));
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusSendMessage: Error Back "
-                      "From Abus MBox Send: Tgt=0x%.08X, link=%d, mbox=%d: "
-                      TRACE_ERR_FMT,
-                      get_huid(i_mProcInfo.tgt), i_send_linkId, i_send_mboxId,
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-
-        // Push this msg_data to TPM
-        err = nodeCommAbusLogNonce(msg_data.value);
-        if (err)
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
-                      "nodeCommAbusLogNonce: "
-                      TRACE_ERR_FMT,
-                      TRACE_ERR_ARGS(err));
-            break;
-        }
-
-    } while( 0 );
-
-    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: "
-              TRACE_ERR_FMT,
-              TRACE_ERR_ARGS(err));
-
-    return err;
-
-} // end of nodeCommAbusSendMessage
-
-
-/**
  *  @brief This function runs the procedure for the master processor on the
  *         master node to send and receive messages over the ABUS to the
  *         master processors on the slave nodes
@@ -549,7 +280,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
     errlHndl_t err = nullptr;
 
     TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusExchangeMaster: mProc=0x%.08X "
-              "to communicate through %d obus connections",
+              "to communicate through %d obus connection(s)",
               get_huid(i_mProcInfo.tgt), i_obus_instances.size());
 
     do
@@ -576,11 +307,30 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
                   expected_peer_linkId, expected_peer_mboxId,
                   l_obus.peerObusInstance);
 
+        // Get random number from TPM
+        msg_format_t msg_data;
+        msg_data.value = 0;
+
+        err = nodeCommAbusGetRandom(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Error Back "
+                      "From nodeCommAbusGetRandom: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+        // Set the send and expected receive LinkIds in the nonce
+        msg_data.origin_linkId = my_linkId;
+        msg_data.receiver_linkId = expected_peer_linkId;
+
+
         // Send a message to a slave
-        err =  nodeCommAbusSendMessage(i_mProcInfo,
+        err =  nodeCommAbusSendMessage(i_mProcInfo.tgt,
+                                       msg_data.value,
                                        my_linkId,
-                                       my_mboxId,
-                                       expected_peer_linkId);
+                                       my_mboxId);
+
         if (err)
         {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
@@ -588,16 +338,37 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
            break;
         }
 
+        // Push this msg_data to TPM
+        err = nodeCommAbusLogNonce(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Error Back From "
+                      "nodeCommAbusLogNonce: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
         // Look for Return Message From The Slave
-        uint64_t actual_recv_linkId = 0;
-        uint64_t actual_recv_mboxId = 0;
-        err = nodeCommAbusRecvMessage(i_mProcInfo,
+        uint8_t actual_recv_linkId = 0;
+        uint8_t actual_recv_mboxId = 0;
+        uint64_t data_recv = 0;
+        err = nodeCommAbusRecvMessage(i_mProcInfo.tgt,
+                                      data_recv,
                                       actual_recv_linkId,
                                       actual_recv_mboxId);
+
+
         if (err)
         {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
                      "nodeCommAbusRecvMessage returned an error");
+
+           // Since we know what bus we expected the message on, call it out
+           addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                 i_mProcInfo.tgt,
+                                 my_linkId,
+                                 err);
            break;
         }
 
@@ -635,22 +406,46 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
                                              actual_recv_linkId,
                                              actual_recv_mboxId));
 
-            // Likely an issue with Processor or its bus
-            err->addHwCallout( i_mProcInfo.tgt,
-                               HWAS::SRCI_PRIORITY_HIGH,
-                               HWAS::DELAYED_DECONFIG,
-                               HWAS::GARD_NULL );
+            // Since we know what bus we expected the message on, call it out
+            addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                  i_mProcInfo.tgt,
+                                  my_linkId,
+                                  err);
 
             // Or HB code failed to do the procedure correctly
             err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_MED);
+                                     HWAS::SRCI_PRIORITY_LOW);
 
+            // Grab FFDC from the target
+            getNodeCommFFDC(NCDD_MODE_ABUS,
+                            i_mProcInfo.tgt,
+                            err);
 
-            // @TODO RTC 184518 - Add bus callout
             break;
         }
 
-    }
+        // Add receiver Link Id to the message data
+        msg_data.value = data_recv;
+        msg_data.receiver_linkId = actual_recv_linkId;
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Msg received "
+                  "= 0x%.16llX. After adding recv link (%d), 0x%.16llX will be "
+                  "stored in the TPM",
+                  data_recv, actual_recv_linkId, msg_data.value);
+
+
+        // Push this msg_data to TPM
+        err = nodeCommAbusLogNonce(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Error Back "
+                      "From nodeCommAbusLogNonce: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
+    } // end of loop on i_obus_instances
+
     if(err)
     {
         break;
@@ -704,15 +499,23 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
                                   my_mboxId);
 
         // First Wait for Message From Master
-        uint64_t actual_linkId = 0;
-        uint64_t actual_mboxId = 0;
-        err = nodeCommAbusRecvMessage(i_mProcInfo,
+        uint8_t actual_linkId = 0;
+        uint8_t actual_mboxId = 0;
+        uint64_t data_recv = 0;
+        err = nodeCommAbusRecvMessage(i_mProcInfo.tgt,
+                                      data_recv,
                                       actual_linkId,
                                       actual_mboxId);
         if (err)
         {
            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: "
                       "nodeCommAbusRecvMessage returned an error");
+
+           // Since we know what bus we expected the message on, call it out
+           addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                 i_mProcInfo.tgt,
+                                 my_linkId,
+                                 err);
            break;
         }
 
@@ -721,7 +524,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         if ((actual_linkId != my_linkId) ||
             (actual_mboxId != my_mboxId))
         {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
                       "Expected Link (%d) Mbox (%d) IDs DO NOT Match the "
                       "Actual Link (%d) Mbox (%d) IDs the message was "
                       "received on",
@@ -751,20 +554,46 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
                                              actual_linkId,
                                              actual_mboxId));
 
-            // Likely an issue with Processor or its bus
-            err->addHwCallout( i_mProcInfo.tgt,
-                               HWAS::SRCI_PRIORITY_HIGH,
-                               HWAS::DELAYED_DECONFIG,
-                               HWAS::GARD_NULL );
+            // Since we know what bus we expected the message on, call it out
+            addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                  i_mProcInfo.tgt,
+                                  my_linkId,
+                                  err);
 
             // Or HB code failed to do the procedure correctly
             err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_MED);
+                                     HWAS::SRCI_PRIORITY_LOW);
+
+            // Grab FFDC from the target
+            getNodeCommFFDC(NCDD_MODE_ABUS,
+                            i_mProcInfo.tgt,
+                            err);
 
-            // @TODO RTC 184518 - Add bus callout
             break;
         }
 
+        // Add receiver Link Id to the message data
+        msg_format_t msg_data;
+        msg_data.value = data_recv;
+        msg_data.receiver_linkId = actual_linkId;
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: Msg received "
+                  "= 0x%.16llX. After adding recv Link Id (%d), 0x%.16llX will "
+                  "be stored in the TPM",
+                  data_recv, actual_linkId, msg_data.value);
+
+
+        // Push this msg_data to TPM
+        err = nodeCommAbusLogNonce(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back "
+                      "From nodeCommAbusLogNonce: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
+
         // Send a message back to the master node
         // Pass in expected peer linkId for nonce logging/extending purposes
         uint8_t peer_linkId = 0;
@@ -773,10 +602,25 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
                                   peer_linkId,
                                   peer_mboxId);
 
-        err =  nodeCommAbusSendMessage(i_mProcInfo,
+        // Get random number from TPM
+        msg_data.value = 0;
+        err = nodeCommAbusGetRandom(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back "
+                      "From nodeCommAbusGetRandom: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
+        // Set the send and expected receive LinkIds in the nonce
+        msg_data.origin_linkId = my_linkId;
+        msg_data.receiver_linkId = my_linkId;
+        err =  nodeCommAbusSendMessage(i_mProcInfo.tgt,
+                                       msg_data.value,
                                        my_linkId,
-                                       my_mboxId,
-                                       peer_linkId);
+                                       my_mboxId);
         if (err)
         {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
@@ -784,6 +628,17 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
            break;
         }
 
+        // Push this msg_data to TPM
+        err = nodeCommAbusLogNonce(msg_data.value);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back From "
+                      "nodeCommAbusLogNonce: "
+                      TRACE_ERR_FMT,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
     } while( 0 );
 
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchangeSlave: "
@@ -1003,6 +858,8 @@ errlHndl_t nodeCommAbusExchange(void)
                       l_peer_path_str);
             continue;
         }
+
+        // Using this OBUS instance so save it off
         l_obusInstance.peerObusInstance = l_peObus.instance;
         l_obusInstance.peerProcInstance = l_peProc.instance;
         l_obusInstance.peerNodeInstance = l_peNode.instance;
@@ -1030,7 +887,7 @@ errlHndl_t nodeCommAbusExchange(void)
                       l_saved_instance.peerObusInstance);
 
                     l_duplicate_found = true;
-                    continue;
+                    break;
                 }
             }
             if (l_duplicate_found == true)
@@ -1041,7 +898,7 @@ errlHndl_t nodeCommAbusExchange(void)
         }
 
         obus_instances.push_back(l_obusInstance);
-        TRACFCOMP(g_trac_nc,"nodeCommAbusExchange: Using masterProc 0x%.08X "
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: Using masterProc 0x%.08X "
                   "OBUS HUID 0x%.08X's peer path %s with obus_instance "
                   "myObusInstance=%d, peer=n%d/p%d/obus%d (vector size=%d)",
                   get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
diff --git a/src/usr/secureboot/node_comm/node_comm_test.C b/src/usr/secureboot/node_comm/node_comm_test.C
index f9ddb372d..e9c6bf116 100644
--- a/src/usr/secureboot/node_comm/node_comm_test.C
+++ b/src/usr/secureboot/node_comm/node_comm_test.C
@@ -67,8 +67,8 @@ errlHndl_t nodeCommXbus2ProcTest(void)
     uint64_t read_data = 0;
     Target* read_tgt = nullptr;
     node_comm_modes_t mode = NCDD_MODE_XBUS;
-    uint64_t linkId = 0;
-    uint64_t mboxId = 0;
+    uint8_t linkId = 0;
+    uint8_t mboxId = 0;
     bool attn_found = false;
 
     do