Add Node Communications Transfer Protocol

This commit adds a more flexible Node-to-Node Transfer Protocol
that will be used for the secureboot/trustedboot algorithm in
multi-node configurations.  The original message transfer has
been updated to use this new transfer protocol.

Change-Id: I12704e8d71d8c0aac52c286160322f9b845a5026
RTC:203641
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/71317
Reviewed-by: Ilya Smirnov <ismirno@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

8 files changed, 738 insertions, 213 deletions

diff --git a/src/usr/secureboot/node_comm/makefile b/src/usr/secureboot/node_comm/makefile
index e34c5b60b..52ca46711 100644
--- a/src/usr/secureboot/node_comm/makefile
+++ b/src/usr/secureboot/node_comm/makefile
@@ -31,6 +31,7 @@ OBJS += node_comm_dd.o
 OBJS += node_comm_test.o
 OBJS += node_comm.o
 OBJS += node_comm_exchange.o
+OBJS += node_comm_transfer.o
 
 VPATH += $(ROOTPATH)/src/usr/targeting/common
 EXTRAINCDIR += $(ROOTPATH)/src/include/usr
diff --git a/src/usr/secureboot/node_comm/node_comm.C b/src/usr/secureboot/node_comm/node_comm.C
index 64b23137a..b05c56505 100644
--- a/src/usr/secureboot/node_comm/node_comm.C
+++ b/src/usr/secureboot/node_comm/node_comm.C
@@ -72,17 +72,19 @@ namespace NODECOMM
  *         ABUS from a processor on another node.
  */
 errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
-                                   uint64_t & o_data,
-                                   uint8_t & o_linkId,
-                                   uint8_t & o_mboxId)
+                                   const uint8_t i_linkId,
+                                   const uint8_t i_mboxId,
+                                   uint64_t & o_data)
 {
     errlHndl_t err = nullptr;
     bool attn_found = false;
+    uint8_t actual_linkId = 0;
+    uint8_t actual_mboxId = 0;
 
     const uint64_t interval_ns = NODE_COMM_POLL_DELAY_NS;
     uint64_t time_polled_ns = 0;
 
-    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: pProc=0x%.08X",
+    TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: pProc=0x%.08X",
               get_huid(i_pProc));
 
     do
@@ -94,8 +96,8 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
         err = nodeCommMapAttn(i_pProc,
                               NCDD_MODE_ABUS,
                               attn_found,
-                              o_linkId,
-                              o_mboxId);
+                              actual_linkId,
+                              actual_mboxId);
         if (err)
         {
             TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: Error Back "
@@ -107,10 +109,10 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
         }
         if (attn_found == true)
         {
-            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
+            TRACUCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
               "nodeCommMapAttn attn_found (%d) for Tgt=0x%.08X, link=%d, "
               "mbox=%d",
-              attn_found, get_huid(i_pProc), o_linkId, o_mboxId);
+              attn_found, get_huid(i_pProc), actual_linkId, actual_mboxId);
             break;
         }
 
@@ -131,7 +133,7 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
              * @userdata2[32:63] Time Interval Between Polls in ns
              * @devdesc          Timed out waiting to receive message over
              *                   ABUS Link Mailbox
-             * @custdesc         Secure Boot failure
+             * @custdesc         Trusted Boot failure
              */
             err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                            MOD_NC_RECV,
@@ -143,8 +145,11 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
                                              NODE_COMM_POLL_DELAY_TOTAL_NS,
                                              interval_ns));
 
-            // Bus Callout will be handled by caller since
-            // it should know expected peer target
+            // Since we know what bus we expected the message on, call it out
+            addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                  i_pProc,
+                                  i_linkId,
+                                  err);
 
             // Or HB code failed to do the procedure correctly
             err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
@@ -172,6 +177,59 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
 
     if (attn_found == true)
     {
+        // Verify that actual receive link/mboxIds were the same as the
+        // expected ones
+        if ((actual_linkId != i_linkId) ||
+            (actual_mboxId != i_mboxId))
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: "
+                      "Expected Link (%d) Mbox (%d) IDs DO NOT Match the "
+                      "Actual Link (%d) Mbox (%d) IDs the message was "
+                      "received on",
+                       i_linkId, i_mboxId,
+                       actual_linkId, actual_mboxId);
+
+            /*@
+             * @errortype
+             * @reasoncode       RC_NCEX_MISMATCH_RECV_LINKS
+             * @moduleid         MOD_NC_RECV
+             * @userdata1        Master Proc Target HUID
+             * @userdata2[0:15]  Expected Link Id to receive message on
+             * @userdata2[16:31] Expected Mailbox Id to receive message on
+             * @userdata2[32:47] Actual Link Id message was received on
+             * @userdata2[48:63] Actual Mailbox Id message was receiveed on
+             * @devdesc          Mismatch between expected and actual Link Mbox
+             *                   Ids a secure ABUS message was received on
+             * @custdesc         Trusted Boot failure
+             */
+            err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                           MOD_NC_RECV,
+                                           RC_NCEX_MISMATCH_RECV_LINKS,
+                                           get_huid(i_pProc),
+                                           FOUR_UINT16_TO_UINT64(
+                                             i_linkId,
+                                             i_mboxId,
+                                             actual_linkId,
+                                             actual_mboxId));
+
+            // Since we know what bus we expected the message on, call it out
+            addNodeCommBusCallout(NCDD_MODE_ABUS,
+                                  i_pProc,
+                                  i_linkId,
+                                  err);
+
+            // Or HB code failed to do the procedure correctly
+            err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                     HWAS::SRCI_PRIORITY_LOW);
+
+            // Grab FFDC from the target
+            getNodeCommFFDC(NCDD_MODE_ABUS,
+                            i_pProc,
+                            err);
+
+            break;
+        }
+
         //  Read message on proc with Link Mailbox found above
         o_data = 0;
         size_t expSize = sizeof(o_data);
@@ -180,15 +238,15 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
                                    &o_data,
                                    reqSize,
                                    DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
-                                                           o_linkId,
-                                                           o_mboxId));
+                                                           actual_linkId,
+                                                           actual_mboxId));
 
         if (err)
         {
             TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
                       "Abus MBox Read: Tgt=0x%.08X, link=%d, mbox=%d: "
                       TRACE_ERR_FMT,
-                      get_huid(i_pProc), o_linkId, o_mboxId,
+                      get_huid(i_pProc), actual_linkId, actual_mboxId,
                       TRACE_ERR_ARGS(err));
             break;
         }
@@ -203,7 +261,8 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
               "Tgt=0x%.08X, link=%d, mbox=%d attn_found=%d: "
               "data=0x%.16llX. "
               TRACE_ERR_FMT,
-              get_huid(i_pProc), o_linkId, o_mboxId, attn_found, o_data,
+              get_huid(i_pProc), actual_linkId, actual_mboxId,
+              attn_found, o_data,
               TRACE_ERR_ARGS(err));
 
     return err;
@@ -216,13 +275,13 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
  *         the current node to a processor on another node.
  */
 errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
-                                   const uint64_t & i_data,
-                                   const uint8_t & i_linkId,
-                                   const uint8_t & i_mboxId)
+                                   const uint64_t i_data,
+                                   const uint8_t i_linkId,
+                                   const uint8_t i_mboxId)
 {
     errlHndl_t err = nullptr;
 
-    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
+    TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
               "to send data=0x%.16llX through linkId=%d mboxId=%d",
               get_huid(i_pProc), i_data, i_linkId, i_mboxId);
 
@@ -253,8 +312,10 @@ errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
 
     } while( 0 );
 
-    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: "
+    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
+              "send data=0x%.16llX through linkId=%d mboxId=%d: "
               TRACE_ERR_FMT,
+              get_huid(i_pProc), i_data, i_linkId, i_mboxId,
               TRACE_ERR_ARGS(err));
 
     return err;
@@ -352,7 +413,7 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
          * @userdata2[32:63] Target HUID FIR was read from
          * @devdesc          Too many attentions were found in
          *                   the Node Comm FIR Register
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NC_MAP_ATTN,
@@ -392,7 +453,7 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
             o_linkId = (bit / 2);
             o_mboxId = (bit % 2);
 
-            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommMapAttn: tgt=0x%X: "
+            TRACUCOMP(g_trac_nc,INFO_MRK"nodeCommMapAttn: tgt=0x%X: "
                       "o_attn_found=%d, o_linkId=%d, mboxId=%d, "
                       TRACE_ERR_FMT,
                       get_huid(i_pProc), o_attn_found, o_linkId, o_mboxId,
@@ -492,7 +553,7 @@ errlHndl_t getObusTrainedLinks(TARGETING::Target* i_pObus,
 /**
  * @brief Add FFDC for the target to an error log
  */
-void getNodeCommFFDC( node_comm_modes_t   i_mode,
+void getNodeCommFFDC( const node_comm_modes_t   i_mode,
                       TARGETING::Target*  i_pProc,
                       errlHndl_t          &io_log)
 {
@@ -556,9 +617,9 @@ void getNodeCommFFDC( node_comm_modes_t   i_mode,
 /**
  * @brief Add a bus callout to an error log
  */
-void addNodeCommBusCallout(node_comm_modes_t i_mode,
+void addNodeCommBusCallout(const node_comm_modes_t i_mode,
                            TARGETING::Target* i_pProc,
-                           uint8_t & i_linkId,
+                           const uint8_t i_linkId,
                            errlHndl_t & io_log,
                            HWAS::callOutPriority i_priority)
 {
diff --git a/src/usr/secureboot/node_comm/node_comm.H b/src/usr/secureboot/node_comm/node_comm.H
index 5a6b7e684..67bf10fb5 100644
--- a/src/usr/secureboot/node_comm/node_comm.H
+++ b/src/usr/secureboot/node_comm/node_comm.H
@@ -108,8 +108,8 @@ enum node_comm_registers_t : uint64_t
  *
  *  @return uint64_t - Calculated Link Mailbox Register Address
  */
-inline uint64_t getLinkMboxRegAddr(uint64_t i_reg,
-                                   node_comm_modes_t i_mode)
+inline uint64_t getLinkMboxRegAddr(const uint64_t i_reg,
+                                   const node_comm_modes_t i_mode)
 {
     return (i_mode == NCDD_MODE_ABUS)
               ? (i_reg + NCDD_ABUS_REG_OFFSET)
@@ -125,7 +125,8 @@ inline uint64_t getLinkMboxRegAddr(uint64_t i_reg,
  *  @return uint64_t - Calculated Link Mailbox Register Base Address
  *  @note   Returned value is Base/XBUS Address - see node_comm_registers_t
  */
-inline uint64_t getLinkMboxReg(uint8_t i_linkId, uint8_t i_mboxId)
+inline uint64_t getLinkMboxReg(const uint8_t i_linkId,
+                               const uint8_t i_mboxId)
 {
     return NCDD_REG_LINK_MBOX_00 + (2*i_linkId) + i_mboxId;
 }
@@ -140,8 +141,8 @@ inline uint64_t getLinkMboxReg(uint8_t i_linkId, uint8_t i_mboxId)
  *
  *  @return void
  */
-inline void getLinkMboxFromObusInstance(uint8_t i_obusInstance,
-                                        uint8_t i_obusRelLink,
+inline void getLinkMboxFromObusInstance(const uint8_t i_obusInstance,
+                                        const uint8_t i_obusRelLink,
                                         uint8_t & o_linkId,
                                         uint8_t & o_mboxId)
 {
@@ -172,7 +173,8 @@ enum node_comm_fir_reg_helpers_t : uint64_t
  *
  *  @return uint64_t - Calculated FIR Register Attention Bit Mask
  */
-inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
+inline uint64_t getLinkMboxFirAttnBit(const uint8_t i_linkId,
+                                      const uint8_t i_mboxId)
 {
     return (NCDD_START_OF_ATTN_BITS >> ((2*i_linkId) + i_mboxId));
 }
@@ -184,24 +186,27 @@ inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
  *
  *  @param[in]  i_pProc  - Processor target to look for attentions on
  *                         Can't be nullptr
+ *  @param[in]  i_linkId - Expected Link Id that received the message
+ *  @param[in]  i_mboxId - Expected Mailbox Id that received the message
  *  @param[out] o_data   - Data received
- *  @param[out] o_linkId - Link Id that received the message
- *  @param[out] o_mboxId - Mailbox Id that received the message
+ *
+ *  @note If message received on different LinkId/MailboxId than i_linkId and
+ *        i_mboxId then an error will be created
  *
  *  @return errlHndl_t Error log handle
  *  @retval nullptr Operation was successful
  *  @retval !nullptr Operation failed with valid error log
  */
 errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
-                                   uint64_t & o_data,
-                                   uint8_t & o_linkId,
-                                   uint8_t & o_mboxId);
+                                   uint8_t i_linkId,
+                                   uint8_t i_mboxId,
+                                   uint64_t & o_data);
 
 /**
  *  @brief This function sends a message over the ABUS from the processor of
  *         the current node to a processor on another node.
  *
- *  @param[in] i_pProc  - Processor target to look for attentions on
+ *  @param[in] i_pProc  - Processor target sending message
  *                        Can't be nullptr
  *  @param[in] i_data   - Data to be sent
  *  @param[in] i_linkId - Link Id Message is sent from
@@ -212,9 +217,9 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
  *  @retval !nullptr Operation failed with valid error log
  */
 errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
-                                   const uint64_t & i_data,
-                                   const uint8_t & i_linkId,
-                                   const uint8_t & i_mboxId);
+                                   uint64_t i_data,
+                                   uint8_t i_linkId,
+                                   uint8_t i_mboxId);
 
 
 /**
@@ -286,7 +291,7 @@ void getNodeCommFFDC(node_comm_modes_t i_mode,
 void addNodeCommBusCallout(
     node_comm_modes_t i_mode,
     TARGETING::Target* i_pProc,
-    uint8_t & i_linkId,
+    uint8_t i_linkId,
     errlHndl_t & io_log,
     HWAS::callOutPriority i_priority = HWAS::SRCI_PRIORITY_HIGH);
 
diff --git a/src/usr/secureboot/node_comm/node_comm_dd.C b/src/usr/secureboot/node_comm/node_comm_dd.C
index dd494ebaf..9e9caf200 100644
--- a/src/usr/secureboot/node_comm/node_comm_dd.C
+++ b/src/usr/secureboot/node_comm/node_comm_dd.C
@@ -109,6 +109,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
     {
 
     // Check other input parameters
+    // @TODO RTC 203642 add check that i_opType is only READ or WRITE
     const auto max_linkId = (mode==NCDD_MODE_ABUS)
                               ? NCDD_MAX_ABUS_LINK_ID
                               : NCDD_MAX_XBUS_LINK_ID;
@@ -136,7 +137,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
          * @userdata1[48:63] MailboxId Value
          * @userdata2        Input Data Pointer
          * @devdesc          Invalid Input Args for Node Comm DD call
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NCDD_PERFORM_OP,
@@ -409,7 +410,7 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
          * @userdata1        Status Register Value
          * @userdata2        Target HUID
          * @devdesc          Sent bit not set in Node Comm status/ctrl register
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NCDD_WRITE,
@@ -501,7 +502,7 @@ errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
              * @userdata1        Status Register Value
              * @userdata2        Target HUID
              * @devdesc          Error found in Node Comm status/ctrl register
-             * @custdesc         Secure Boot failure
+             * @custdesc         Trusted Boot failure
              */
             err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                            MOD_NCDD_CHECK_FOR_ERRORS,
@@ -597,7 +598,7 @@ errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args,
                  * @userdata2[0:31]  Status/Control Register Address
                  * @userdata2[32:63] Target HUID
                  * @devdesc          Timed out waiting for command complete.
-                 * @custdesc         Secure Boot failure
+                 * @custdesc         Trusted Boot failure
                  */
                 err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                               MOD_NCDD_WAIT_FOR_CMD_COMP,
diff --git a/src/usr/secureboot/node_comm/node_comm_exchange.C b/src/usr/secureboot/node_comm/node_comm_exchange.C
index 9f7938dc6..5383056b6 100644
--- a/src/usr/secureboot/node_comm/node_comm_exchange.C
+++ b/src/usr/secureboot/node_comm/node_comm_exchange.C
@@ -54,6 +54,7 @@
 #include <config.h>
 
 #include "node_comm.H"
+#include "node_comm_transfer.H"
 
 // ----------------------------------------------
 // Defines
@@ -151,7 +152,7 @@ errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
          * @userdata1        <Unused>
          * @userdata2        <Unused>
          * @devdesc          No functional TPMs were found
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_PREDICTIVE,
                                        MOD_NCEX_GET_RANDOM,
@@ -167,7 +168,9 @@ errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
     }
 
     // Use first of functional TPM target list
+    // @TODO RTC 203642 Update this to use Primary TPM
     tpm_tgt = tpmTargetList[0];
+
     // This function call requires the CONFIG check for compilation purposes,
     // but no extra error handling is needed as it should not have gotten this
     // far if CONFIG_TPMDD wasn't set
@@ -286,6 +289,9 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
               "to communicate through %d obus connection(s)",
               get_huid(i_mProcInfo.tgt), i_obus_instances.size());
 
+    // Pointer to buffer used for receiving data from nodeCommTransferRecv()
+    uint8_t * data_rcv_buffer = nullptr;
+
     do
     {
 
@@ -330,18 +336,21 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         msg_data.origin_linkId = my_linkId;
         msg_data.receiver_linkId = expected_peer_linkId;
 
-
         // Send a message to a slave
-        err =  nodeCommAbusSendMessage(i_mProcInfo.tgt,
-                                       msg_data.value,
-                                       my_linkId,
-                                       my_mboxId);
+        size_t tmp_size = sizeof(msg_data.value);
+        err = nodeCommTransferSend(i_mProcInfo.tgt,
+                                   my_linkId,
+                                   my_mboxId,
+                                   NCT_TRANSFER_SBID,
+                                   reinterpret_cast<uint8_t*>
+                                     (&(msg_data.value)),
+                                   tmp_size);
 
         if (err)
         {
-           TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
-                     "nodeCommAbusSendMessage returned an error");
-           break;
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
+                      "nodeCommAbusTransferSend returned an error");
+            break;
         }
 
         // Push this msg_data to TPM
@@ -356,88 +365,37 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         }
 
         // Look for Return Message From The Slave
-        uint8_t actual_recv_linkId = 0;
-        uint8_t actual_recv_mboxId = 0;
-        uint64_t data_recv = 0;
-        err = nodeCommAbusRecvMessage(i_mProcInfo.tgt,
-                                      data_recv,
-                                      actual_recv_linkId,
-                                      actual_recv_mboxId);
-
-
-        if (err)
+        size_t data_rcv_size = 0;
+        if (data_rcv_buffer != nullptr)
         {
-           TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
-                     "nodeCommAbusRecvMessage returned an error");
-
-           // Since we know what bus we expected the message on, call it out
-           addNodeCommBusCallout(NCDD_MODE_ABUS,
-                                 i_mProcInfo.tgt,
-                                 my_linkId,
-                                 err);
-           break;
+            free(data_rcv_buffer);
+            data_rcv_buffer = nullptr;
         }
-
-        // Verify that actual receive link/mboxIds were the same as the
-        // ones this node sent from
-        if ((my_linkId != actual_recv_linkId) ||
-            (my_mboxId != actual_recv_mboxId))
+        err = nodeCommTransferRecv(i_mProcInfo.tgt,
+                                   my_linkId,
+                                   my_mboxId,
+                                   NCT_TRANSFER_SBID,
+                                   data_rcv_buffer,
+                                   data_rcv_size);
+        if (err)
         {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: "
-                      "Expected Link (%d) Mbox (%d) IDs DO NOT Match the "
-                      "Actual Link (%d) Mbox (%d) IDs the return message used",
-                       my_linkId, my_mboxId,
-                       actual_recv_linkId, actual_recv_mboxId);
-
-            /*@
-             * @errortype
-             * @reasoncode       RC_NCEX_MISMATCH_RECV_LINKS
-             * @moduleid         MOD_NCEX_MASTER
-             * @userdata1        Master Proc Target HUID
-             * @userdata2[0:15]  Expected Link Id to receive message on
-             * @userdata2[16:31] Expected Mailbox Id to receive message on
-             * @userdata2[32:47] Acutal Link Id message was received on
-             * @userdata2[48:63] Actual Mailbox Id message was receiveed on
-             * @devdesc          Mismatch between expected and actual Link Mbox
-             *                   Ids a secure ABUS message was received on
-             * @custdesc         Secure Boot failure
-             */
-            err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                           MOD_NCEX_MASTER,
-                                           RC_NCEX_MISMATCH_RECV_LINKS,
-                                           get_huid(i_mProcInfo.tgt),
-                                           FOUR_UINT16_TO_UINT64(
-                                             my_linkId,
-                                             my_mboxId,
-                                             actual_recv_linkId,
-                                             actual_recv_mboxId));
-
-            // Since we know what bus we expected the message on, call it out
-            addNodeCommBusCallout(NCDD_MODE_ABUS,
-                                  i_mProcInfo.tgt,
-                                  my_linkId,
-                                  err);
-
-            // Or HB code failed to do the procedure correctly
-            err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_LOW);
-
-            // Grab FFDC from the target
-            getNodeCommFFDC(NCDD_MODE_ABUS,
-                            i_mProcInfo.tgt,
-                            err);
-
-            break;
+           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: "
+                      "nodeCommTransferRecv returned an error");
+           break;
         }
+        // If no err is returned, data_rcv_buffer should be valid, but do a
+        // sanity check here to be certain
+        assert(data_rcv_buffer!=nullptr,"nodeCommAbusExchangeMaster: data_rcv_buffer returned as nullptr");
 
         // Add receiver Link Id to the message data
-        msg_data.value = data_recv;
-        msg_data.receiver_linkId = actual_recv_linkId;
-        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Msg received "
-                  "= 0x%.16llX. After adding recv link (%d), 0x%.16llX will be "
+        // @TODO RTC 203642 Check that data_rcv_size == sizeof(uint64_t)
+        // here and in other places where SBID is handled
+        memcpy(&(msg_data.value), data_rcv_buffer, data_rcv_size);
+        msg_data.receiver_linkId = my_linkId;
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Msg received, "
+                  "after adding recv link (%d), 0x%.16llX will be "
                   "stored in the TPM",
-                  data_recv, actual_recv_linkId, msg_data.value);
-
+                  my_linkId, msg_data.value);
 
         // Push this msg_data to TPM
         err = nodeCommAbusLogNonce(msg_data.value);
@@ -457,8 +415,16 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         break;
     }
 
+    // @TODO RTC 203642 Do Another Loop for Quotes
+
     } while( 0 );
 
+    if (data_rcv_buffer != nullptr)
+    {
+        free(data_rcv_buffer);
+        data_rcv_buffer = nullptr;
+    }
+
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchangeMaster: "
               TRACE_ERR_FMT,
               TRACE_ERR_ARGS(err));
@@ -495,6 +461,9 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
               i_obus_instance.peerProcInstance,
               i_obus_instance.peerObusInstance);
 
+    // Pointer to buffer used for receiving data from nodeCommTransferRecv()
+    uint8_t * data_rcv_buffer = nullptr;
+
     do
     {
         // Used for check that right node indicated itself as master
@@ -505,89 +474,40 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
                                     my_linkId,
                                     my_mboxId);
 
+
         // First Wait for Message From Master
-        uint8_t actual_linkId = 0;
-        uint8_t actual_mboxId = 0;
-        uint64_t data_recv = 0;
-        err = nodeCommAbusRecvMessage(i_mProcInfo.tgt,
-                                      data_recv,
-                                      actual_linkId,
-                                      actual_mboxId);
+        if (data_rcv_buffer != nullptr)
+        {
+            free(data_rcv_buffer);
+            data_rcv_buffer = nullptr;
+        }
+        size_t data_rcv_size = 0;
+        err = nodeCommTransferRecv(i_mProcInfo.tgt,
+                                   my_linkId,
+                                   my_mboxId,
+                                   NCT_TRANSFER_SBID,
+                                   data_rcv_buffer,
+                                   data_rcv_size);
         if (err)
         {
            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: "
-                      "nodeCommAbusRecvMessage returned an error");
-
-           // Since we know what bus we expected the message on, call it out
-           addNodeCommBusCallout(NCDD_MODE_ABUS,
-                                 i_mProcInfo.tgt,
-                                 my_linkId,
-                                 err);
+                      "nodeCommTransferRecv returned an error");
            break;
         }
+        // If no err is returned, data_rcv_buffer should be valid, but do a
+        // sanity check here to be certain
+        assert(data_rcv_buffer!=nullptr,"nodeCommAbusExchangeSlave: data_rcv_buffer returned as nullptr");
 
-        // Verify that actual receive link/mboxIds were the same as the
-        // expected ones
-        if ((actual_linkId != my_linkId) ||
-            (actual_mboxId != my_mboxId))
-        {
-            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
-                      "Expected Link (%d) Mbox (%d) IDs DO NOT Match the "
-                      "Actual Link (%d) Mbox (%d) IDs the message was "
-                      "received on",
-                       my_linkId, my_mboxId,
-                       actual_linkId, actual_mboxId);
-
-            /*@
-             * @errortype
-             * @reasoncode       RC_NCEX_MISMATCH_RECV_LINKS
-             * @moduleid         MOD_NCEX_SLAVE
-             * @userdata1        Master Proc Target HUID
-             * @userdata2[0:15]  Expected Link Id to receive message on
-             * @userdata2[16:31] Expected Mailbox Id to receive message on
-             * @userdata2[32:47] Actual Link Id message was received on
-             * @userdata2[48:63] Actual Mailbox Id message was receiveed on
-             * @devdesc          Mismatch between expected and actual Link Mbox
-             *                   Ids a secure ABUS message was received on
-             * @custdesc         Secure Boot failure
-             */
-            err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                           MOD_NCEX_SLAVE,
-                                           RC_NCEX_MISMATCH_RECV_LINKS,
-                                           get_huid(i_mProcInfo.tgt),
-                                           FOUR_UINT16_TO_UINT64(
-                                             my_linkId,
-                                             my_mboxId,
-                                             actual_linkId,
-                                             actual_mboxId));
-
-            // Since we know what bus we expected the message on, call it out
-            addNodeCommBusCallout(NCDD_MODE_ABUS,
-                                  i_mProcInfo.tgt,
-                                  my_linkId,
-                                  err);
-
-            // Or HB code failed to do the procedure correctly
-            err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                     HWAS::SRCI_PRIORITY_LOW);
-
-            // Grab FFDC from the target
-            getNodeCommFFDC(NCDD_MODE_ABUS,
-                            i_mProcInfo.tgt,
-                            err);
-
-            break;
-        }
 
         // Add receiver Link Id to the message data
         msg_format_t msg_data;
-        msg_data.value = data_recv;
-        msg_data.receiver_linkId = actual_linkId;
-        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: Msg received "
-                  "= 0x%.16llX. After adding recv Link Id (%d), 0x%.16llX will "
-                  "be stored in the TPM",
-                  data_recv, actual_linkId, msg_data.value);
+        memcpy(&(msg_data.value), data_rcv_buffer, data_rcv_size);
+        msg_data.receiver_linkId = my_linkId;
 
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: Msg received, "
+                  "after adding recv Link Id (%d), 0x%.16llX will "
+                  "be stored in the TPM",
+                  my_linkId, msg_data.value);
 
         // Push this msg_data to TPM
         err = nodeCommAbusLogNonce(msg_data.value);
@@ -600,7 +520,6 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
             break;
         }
 
-
         // Send a message back to the master node
         // Pass in expected peer linkId for nonce logging/extending purposes
         uint8_t peer_linkId = 0;
@@ -626,15 +545,22 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         // Set the send and expected receive LinkIds in the nonce
         msg_data.origin_linkId = my_linkId;
         msg_data.receiver_linkId = my_linkId;
-        err =  nodeCommAbusSendMessage(i_mProcInfo.tgt,
-                                       msg_data.value,
-                                       my_linkId,
-                                       my_mboxId);
+
+        // Send a message to a slave
+        size_t tmp_size = sizeof(msg_data.value);
+        err = nodeCommTransferSend(i_mProcInfo.tgt,
+                                   my_linkId,
+                                   my_mboxId,
+                                   NCT_TRANSFER_SBID,
+                                   reinterpret_cast<uint8_t*>
+                                     (&(msg_data.value)),
+                                   tmp_size);
+
         if (err)
         {
-           TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
-                      "nodeCommAbusSendMessage returned an error");
-           break;
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
+                      "nodeCommAbusTransferSend returned an error");
+            break;
         }
 
         // Push this msg_data to TPM
@@ -650,6 +576,13 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
 
     } while( 0 );
 
+    if (data_rcv_buffer != nullptr)
+    {
+        free(data_rcv_buffer);
+        data_rcv_buffer = nullptr;
+    }
+
+
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchangeSlave: "
               TRACE_ERR_FMT,
               TRACE_ERR_ARGS(err));
@@ -755,7 +688,7 @@ errlHndl_t nodeCommAbusExchange(void)
          * @userdata2        <Unused>
          * @devdesc          Master Proc's ATTR_PHYS_PATH is invalid as it
          *                   doesn't have either a NODE or PROC elemenent
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NCEX_MAIN,
@@ -976,7 +909,7 @@ errlHndl_t nodeCommAbusExchange(void)
          * @userdata2[32:63] Total Number Of Nodes
          * @devdesc          When processing the OBUS Peer paths, the wrong
          *                   count of valid paths was found
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NCEX_MAIN,
diff --git a/src/usr/secureboot/node_comm/node_comm_test.C b/src/usr/secureboot/node_comm/node_comm_test.C
index e9c6bf116..acf370a02 100644
--- a/src/usr/secureboot/node_comm/node_comm_test.C
+++ b/src/usr/secureboot/node_comm/node_comm_test.C
@@ -167,7 +167,7 @@ errlHndl_t nodeCommXbus2ProcTest(void)
          * @userdata2[32:63] Target HUID data was sent from
          * @devdesc          No Attention was found after sending data in
          *                   XBUS Node Comm Test
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NC_XBUS_TEST,
@@ -242,7 +242,7 @@ errlHndl_t nodeCommXbus2ProcTest(void)
          * @userdata2[32:63] Target HUID data was read from
          * @devdesc          No Attention was found after sending data in
          *                   Xbus Node Comm Test
-         * @custdesc         Secure Boot failure
+         * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                        MOD_NC_XBUS_TEST,
diff --git a/src/usr/secureboot/node_comm/node_comm_transfer.C b/src/usr/secureboot/node_comm/node_comm_transfer.C
new file mode 100644
index 000000000..daa6b6cbb
--- /dev/null
+++ b/src/usr/secureboot/node_comm/node_comm_transfer.C
@@ -0,0 +1,378 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/secureboot/node_comm/node_comm_transfer.C $           */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2019                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+
+// ----------------------------------------------
+// Includes
+// ----------------------------------------------
+#include <config.h>
+#include <time.h>
+#include <devicefw/userif.H>
+#include <trace/interface.H>
+#include <scom/centaurScomCache.H> // for TRACE_ERR_FMT, TRACE_ERR_ARGS
+#include <secureboot/nodecommif.H>
+#include "node_comm.H"
+#include "node_comm_transfer.H"
+
+// ----------------------------------------------
+// Defines
+// ----------------------------------------------
+
+
+namespace SECUREBOOT
+{
+
+namespace NODECOMM
+{
+
+
+errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
+                                const uint8_t i_linkId,
+                                const uint8_t i_mboxId,
+                                node_comm_transfer_types_t i_transferType,
+                                const uint8_t * i_data,
+                                const size_t i_dataSize)
+{
+    errlHndl_t err = nullptr;
+
+    // Determine how many messages need to be sent:
+    // msg sequence 0: Initiation Message
+    // msg sequence 1..N: 8-byte data messages
+    size_t msg_seq = 0;
+    size_t total_data_msgs = (i_dataSize + (sizeof(uint64_t)-1))
+                             /sizeof(uint64_t);
+
+    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommTransferSend: iProc=0x%.08X "
+              "to send %d bytes of data through linkId=%d mboxId=%d over "
+              "%d data messages",
+              get_huid(i_pProc), i_dataSize, i_linkId, i_mboxId, total_data_msgs);
+
+    do
+    {
+        // Keep track of data sent
+        size_t bytes_sent = 0;
+        size_t bytes_left = i_dataSize;
+
+        // Loop of sending messages (initiation and data) and receiving ACKs
+        for ( ; msg_seq <= total_data_msgs; ++msg_seq)
+        {
+            TRACUCOMP(g_trac_nc,INFO_MRK"nodeCommTransferSend: loop start: "
+                      "seq = %d (msg %d of %d)",
+                      msg_seq, msg_seq+1, total_data_msgs);
+
+            uint64_t data = 0;
+
+            if (msg_seq==0)
+            {
+                // Send Initiaion Message
+                node_comm_msg_format_t send_msg;
+                send_msg.value = 0;  // Clear out the data
+
+                // @TODO RTC 203642 Fix sendingNode and recving Node
+                send_msg.sendingNode = 0;
+                send_msg.recvingNode = 1;
+
+                send_msg.msgType = i_transferType;
+                send_msg.msgSeqNum = msg_seq;
+                send_msg.totalDataMsgs = total_data_msgs;
+                send_msg.totalDataSize = i_dataSize;
+
+                data = send_msg.value;
+            }
+            else
+            {
+                // Send Data
+                // Write a max of 8 bytes at a time
+                size_t loop_data_length =
+                            (bytes_left > sizeof(uint64_t)) ?
+                                sizeof(uint64_t) :
+                                bytes_left;
+
+                data=0;
+                memcpy(&data,
+                       i_data + bytes_sent,
+                       loop_data_length);
+
+                // Update amount of data left to be written
+                bytes_sent += loop_data_length;
+                bytes_left -= loop_data_length;
+            }
+            TRACDCOMP(g_trac_nc,INFO_MRK"nodeCommTransferSend: "
+                      "msg %d: sending data=0x%.16llX (bytes_sent=%d, "
+                      "bytes_left=%d)",
+                      msg_seq, data, bytes_sent, bytes_left);
+
+            err = nodeCommAbusSendMessage(i_pProc,
+                                          data,
+                                          i_linkId,
+                                          i_mboxId);
+
+            if (err)
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferSend: "
+                          "nodeCommAbusSendMessage returned an error: "
+                          "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                          TRACE_ERR_FMT,
+                          get_huid(i_pProc), data, i_linkId, i_mboxId,
+                          TRACE_ERR_ARGS(err));
+                break;
+            }
+
+            // Get ACK
+            uint64_t data_recv = 0;
+            err = nodeCommAbusRecvMessage(i_pProc,
+                                          i_linkId,
+                                          i_mboxId,
+                                          data_recv);
+            if (err)
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferSend: "
+                          "nodeCommAbusRecvMessage returned an error: "
+                          "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                          TRACE_ERR_FMT,
+                          get_huid(i_pProc), data, i_linkId, i_mboxId,
+                          TRACE_ERR_ARGS(err));
+
+                break;
+            }
+            TRACDCOMP(g_trac_nc,INFO_MRK"nodeCommTransferSend: "
+                      "msg %d: receiving ACK = 0x%.16llX",
+                      msg_seq, data_recv);
+
+
+
+            // @TODO RTC 203642 Check that ACK is
+            // -- from the right node
+            // -- from right linkId/mboxId
+            // -- the right type of ACK
+
+        }  // end of for-loop of messages
+
+        if (err)
+        {
+            break;
+        }
+
+    } while( 0 );
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommTransferSend: "
+              TRACE_ERR_FMT,
+              TRACE_ERR_ARGS(err));
+
+    return err;
+
+} // end of nodeCommTransferSend
+
+
+
+errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
+                                const uint8_t i_linkId,
+                                const uint8_t i_mboxId,
+                                node_comm_transfer_types_t i_transferType,
+                                uint8_t*& o_data,
+                                size_t & o_dataSize)
+{
+    errlHndl_t err = nullptr;
+
+    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommTransferRecv: i_pProc=0x%.08X "
+              "expecting messages of type 0x%.02x from linkId=%d mboxId=%d",
+              get_huid(i_pProc), i_transferType, i_linkId, i_mboxId);
+
+    // Clear the output variables to be safe
+    o_data = nullptr;
+    o_dataSize = 0;
+
+    do
+    {
+
+        // Wait for Message
+        uint64_t data_recv = 0;
+        err = nodeCommAbusRecvMessage(i_pProc,
+                                      i_linkId,
+                                      i_mboxId,
+                                      data_recv);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferRecv: "
+                      "nodeCommAbusRecvMessage returned an error: "
+                      "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                      TRACE_ERR_FMT,
+                      get_huid(i_pProc), data_recv, i_linkId, i_mboxId,
+                      TRACE_ERR_ARGS(err));
+
+            break;
+        }
+        node_comm_msg_format_t init_msg = {.value = data_recv};
+        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommTransferRecv: "
+                  "Initiation Message 0x%.16llX: sN=%d, rN=%d, msgType=0x%X, "
+                  "msgSeqNum=%d, totalDataMsgs=%d.totalDataSize=%d",
+                  init_msg.value, init_msg.sendingNode, init_msg.recvingNode,
+                  init_msg.msgType, init_msg.msgSeqNum, init_msg.totalDataMsgs,
+                  init_msg.totalDataSize);
+
+        // @TODO RTC 203642 Add the following checks:
+        // - check nodes
+        // - init_msg.msgSeqNum should be 0 for initiation message
+        // - check that expected transfer type is received
+
+
+        // Reply with ACK:
+        node_comm_msg_format_t ack_msg = {.value = 0};
+        ack_msg.value = 0;
+
+        // @TODO RTC 203642 Fix sending node Ids
+        // Always set these for ACK of Initiation:
+        ack_msg.msgSeqNum = 0;
+        ack_msg.msgType = NCT_ACK_OF_INTIATION;
+
+        // ACK total number of expected data messages
+        ack_msg.totalDataMsgs = init_msg.totalDataMsgs;
+
+        // Send ACK message
+        err = nodeCommAbusSendMessage(i_pProc,
+                                      ack_msg.value,
+                                      i_linkId,
+                                      i_mboxId);
+
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferRecv: "
+                      "nodeCommAbusSendMessage returned an error: "
+                      "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                      TRACE_ERR_FMT,
+                      get_huid(i_pProc), ack_msg.value, i_linkId, i_mboxId,
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+
+
+        // msg sequence 0: Initiation Message
+        // msg sequence 1..N: 8-byte data messages
+        size_t msg_seq = 1;
+        size_t total_data_msgs = init_msg.totalDataMsgs;
+
+        // Keep track of data read
+        size_t bytes_read = 0;
+        size_t bytes_left = init_msg.totalDataSize;
+
+        // Create output buffer to store the data
+        uint8_t * data_buffer = reinterpret_cast<uint8_t*>
+                                  (malloc(init_msg.totalDataSize));
+
+
+        // Loop of receiving data messages and sending ACKs
+        for ( ; msg_seq <= total_data_msgs; ++msg_seq)
+        {
+            // Wait for Data Message
+            data_recv = 0;
+            err = nodeCommAbusRecvMessage(i_pProc,
+                                          i_linkId,
+                                          i_mboxId,
+                                          data_recv);
+            if (err)
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferRecv: "
+                          "nodeCommAbusRecvMessage returned an error: "
+                          "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                          TRACE_ERR_FMT,
+                          get_huid(i_pProc), data_recv, i_linkId, i_mboxId,
+                          TRACE_ERR_ARGS(err));
+
+                    break;
+            }
+            TRACDCOMP(g_trac_nc,INFO_MRK"nodeCommTransferRecv: "
+                      "Data Msg Recvd: %d: 0x%.16llX",
+                      msg_seq, data_recv);
+
+            size_t loop_bytes_read = (bytes_left > sizeof(uint64_t)) ?
+                                        sizeof(uint64_t) :
+                                        bytes_left;
+
+            memcpy(data_buffer+bytes_read,
+                   &data_recv,
+                   loop_bytes_read);
+
+            data_recv=0; // clear for next loop
+
+            bytes_read += loop_bytes_read;
+            bytes_left -= loop_bytes_read;
+
+            // Send Data ACK
+            ack_msg.msgType = NCT_ACK_OF_DATA;
+            ack_msg.msgSeqNum = msg_seq;
+            ack_msg.totalDataSize = bytes_read;
+
+            TRACDCOMP(g_trac_nc,INFO_MRK"nodeCommTransferRecv: "
+                      "Data Msg ACK: %d: 0x%.16llX (bytes_read=%d, "
+                      "bytes_left=%d)",
+                      msg_seq, ack_msg.value, bytes_read, bytes_left);
+
+            err = nodeCommAbusSendMessage(i_pProc,
+                                          ack_msg.value,
+                                          i_linkId,
+                                          i_mboxId);
+
+            if (err)
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferRecv: "
+                          "nodeCommAbusSendMessage returned an error: "
+                          "Tgt=0x%.08X, data=0x%.16llX, link=%d, mbox=%d: "
+                          TRACE_ERR_FMT,
+                          get_huid(i_pProc), ack_msg.value, i_linkId, i_mboxId,
+                          TRACE_ERR_ARGS(err));
+                break;
+            }
+        }  // end of for-loop for receiving data messages
+
+        if (err)
+        {
+            free(data_buffer);
+            o_data=nullptr;
+            o_dataSize=0;
+            break;
+        }
+        else
+        {
+            o_data = data_buffer;
+            o_dataSize = init_msg.totalDataSize;
+        }
+
+
+    } while( 0 );
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommTransferRecv: "
+              "o_data=%p, o_dataSize=%d. "
+              TRACE_ERR_FMT,
+              o_data, o_dataSize,
+              TRACE_ERR_ARGS(err));
+
+    return err;
+
+} // end of nodeCommTransferRecv
+
+} // end NODECOMM namespace
+
+} // end SECUREBOOT namespace
+
+
diff --git a/src/usr/secureboot/node_comm/node_comm_transfer.H b/src/usr/secureboot/node_comm/node_comm_transfer.H
new file mode 100644
index 000000000..f64b1e387
--- /dev/null
+++ b/src/usr/secureboot/node_comm/node_comm_transfer.H
@@ -0,0 +1,146 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/secureboot/node_comm/node_comm_transfer.H $           */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2019                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#ifndef __NODE_COMM_TRANSFER_H
+#define __NODE_COMM_TRANSFER_H
+
+// ----------------------------------------------
+// Includes
+// ----------------------------------------------
+#include <config.h>
+#include "node_comm.H"
+
+// ----------------------------------------------
+// Defines
+// ----------------------------------------------
+
+
+namespace SECUREBOOT
+{
+
+namespace NODECOMM
+{
+
+
+/*
+ * @brief Node Comm Transfer Types - descriptions of messages sent between nodes
+ */
+enum node_comm_transfer_types_t : uint8_t
+{
+    // Size of 1 byte due to its inclusion in node_comm_msg_format_t
+
+    // Invalid Type for node_comm_msg_format_t initialization
+    NCT_INVALID_TYPE               =  0x00,
+
+    // Types of ACKs (Ackowledgments)
+    NCT_ACK_OF_INTIATION           =  0x01,
+    NCT_ACK_OF_DATA                =  0x02,
+    // @TODO RTC 203642: ACK of non-support and shutting down the channel?
+
+    // Message Transfer Types
+    NCT_TRANSFER_SBID              =  0x10,
+    NCT_TRANSFER_QUOTE_REQUEST     =  0x11,
+    NCT_TRANSFER_QUOTE_RESPONSE    =  0x12,
+};
+
+/**
+ * @brief Node Comm Message Format
+ *
+ * @note: msg sequence 0: Initiation Message
+ *        msg sequence 1..N: 8-byte data messages
+ *
+ */
+union node_comm_msg_format_t
+{
+    uint64_t value;
+    struct
+    {
+        uint64_t sendingNode   :  4; // Sending Node
+        uint64_t recvingNode   :  4; // Receiving Node
+        uint64_t msgType       :  8; // Msg Type (node_comm_transfer_types_t)
+        uint64_t msgSeqNum     : 16; // Msg Sequence Number
+        uint64_t totalDataMsgs : 16; // Total Number of Expected Data Messages
+        uint64_t totalDataSize : 16; // Total Size of Data in bytes
+    } PACKED;
+};
+
+
+/**
+ *  @brief This function sends multiple messages with data over the ABUS from
+           the processor of the current node to a processor on another node.
+ *
+ *  @param[in] i_pProc         - Processor target that is sending the data
+ *                               Can't be nullptr
+ *  @param[in] i_linkId        - Link Id Message is sent from
+ *  @param[in] i_mboxId        - Mailbox Id Message is sent from
+ *  @param[in] i_transferType  - Transfer Message Type to send
+ *  @param[in] i_data          - Data to be sent
+ *  @param[in] i_dataSize      - Size of Data to be sent in bytes
+ *
+ *  @return errlHndl_t Error log handle
+ *  @retval nullptr Operation was successful
+ *  @retval !nullptr Operation failed with valid error log
+ */
+errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
+                                uint8_t i_linkId,
+                                uint8_t i_mboxId,
+                                node_comm_transfer_types_t i_transferType,
+                                const uint8_t * i_data,
+                                size_t i_dataSize);
+
+/**
+ *  @brief This function waits to receive multiple messages over the ABUS
+ *         from a processor on another node.
+ *
+ *  @param[in]  i_pProc         - Processor target to receive messages on
+ *                                Can't be nullptr
+ *  @param[in]  i_linkId        - Link Id Message is expected from
+ *  @param[in]  i_mboxId        - Mailbox Id Message is expected from
+ *  @param[in]  i_transferType  - Expected Transfer Message Type to receive
+ *                                Failure if different type is received
+ *  @param[out] o_data          - Data received
+ *  @param[out] o_dataSize      - Size of Data received
+ *
+ *  @note - While this function will dynamically create o_data buffer of size
+ *          o_dataSize, the caller is responsible for deleting this buffer.
+ *          If an error is returned, o_data will be nullptr and o_dataSize
+ *          will be zero.
+ *
+ *  @return errlHndl_t Error log handle
+ *  @retval nullptr Operation was successful
+ *  @retval !nullptr Operation failed with valid error log
+ */
+errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
+                                uint8_t i_linkId,
+                                uint8_t i_mboxId,
+                                node_comm_transfer_types_t i_transferType,
+                                uint8_t*& o_data,
+                                size_t & o_dataSize);
+
+} // end NODECOMM namespace
+
+} // end SECUREBOOT namespace
+
+#endif // End __NODE_COMM_TRANSFER_H
+