Secureboot: Enhance Error Callouts For New Multinode Trustedboot Transfer

This commit adds many checks to callout potential issues with the new
multinode trustedboot transfer protocol.  It also improves some
TPM-related traces.

Change-Id: Ice3f8be0668cc63321eeb2562bb8ffe610284b6a
RTC:203642
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/72363
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Ilya Smirnov <ismirno@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

7 files changed, 336 insertions, 74 deletions

diff --git a/src/include/usr/secureboot/secure_reasoncodes.H b/src/include/usr/secureboot/secure_reasoncodes.H
index 51947a7f7..d121fc7b9 100644
--- a/src/include/usr/secureboot/secure_reasoncodes.H
+++ b/src/include/usr/secureboot/secure_reasoncodes.H
@@ -68,6 +68,8 @@ namespace SECUREBOOT
         MOD_NC_GEN_SLAVE_RESPONSE           = 0x2B,
         MOD_NC_GEN_MASTER_REQUEST           = 0x2C,
         MOD_NC_PROCESS_SLAVE_QUOTE          = 0x2D,
+        MOD_NCT_SEND                        = 0x2E,
+        MOD_NCT_RECEIVE                     = 0x2F,
    };
 
     enum SECUREReasonCode
@@ -98,7 +100,7 @@ namespace SECUREBOOT
         RC_COULD_NOT_ALLOCATE_SMF_MEM      = SECURE_COMP_ID | 0x17,
         RC_ALLOCATED_NE_REQUESTED          = SECURE_COMP_ID | 0x18,
 
-        // Use 0x20-0x2F range for Node Communications
+        // Use 0x20-0x3F range for Node Communications
         RC_NCDD_HW_ERROR_FOUND             = SECURE_COMP_ID | 0x20,
         RC_NCDD_CMD_COMP_TIMEOUT           = SECURE_COMP_ID | 0x21,
         RC_NC_DATA_MISCOMPARE              = SECURE_COMP_ID | 0x22,
@@ -115,6 +117,11 @@ namespace SECUREBOOT
         RC_NC_NO_PRIMARY_TPM_LOG           = SECURE_COMP_ID | 0x2D,
         RC_NC_BAD_MASTER_EYE_CATCH         = SECURE_COMP_ID | 0x2E,
         RC_NC_BAD_SLAVE_QUOTE              = SECURE_COMP_ID | 0x2F,
+        RC_NCDD_INVALID_OP_TYPE            = SECURE_COMP_ID | 0x30,
+        RC_NCT_TYPE_SIZE_MISMATCH          = SECURE_COMP_ID | 0x31,
+        RC_NCT_ACK_MISMATCH                = SECURE_COMP_ID | 0x32,
+        RC_NCT_INITIATION_MISMATCH         = SECURE_COMP_ID | 0x33,
+        RC_NCEX_NO_FUNCTIONAL_PRIMARY_TPM  = SECURE_COMP_ID | 0x34,
 
         // Reason codes 0xA0 - 0xEF reserved for trustedboot_reasoncodes.H
     };
diff --git a/src/usr/i2c/tpmdd.C b/src/usr/i2c/tpmdd.C
index 11b14aa80..a6f96514b 100755
--- a/src/usr/i2c/tpmdd.C
+++ b/src/usr/i2c/tpmdd.C
@@ -693,7 +693,8 @@ bool tpmPresence (TARGETING::Target* i_pTpm)
 
             ERRORLOG::ErrlUserDetailsTarget(i_pTpm).addToLog(pError);
 
-            const auto plid = pError->plid();
+            const auto original_eid  = pError->eid();
+            const auto original_plid = pError->plid();
             pError->setSev(ERRORLOG::ERRL_SEV_UNRECOVERABLE);
             errlCommit(pError,TPMDD_COMP_ID);
 
@@ -721,9 +722,16 @@ bool tpmPresence (TARGETING::Target* i_pTpm)
                 get_huid(i_pTpm),
                 0,
                 ERRORLOG::ErrlEntry::NO_SW_CALLOUT);
-            pError->plid(plid);
+            pError->plid(original_plid);
             ERRORLOG::ErrlUserDetailsTarget(i_pTpm).addToLog(pError);
 
+            TRACFCOMP(g_trac_tpmdd, ERR_MRK
+                "tpmPresence: Due to Error eid=0x%.8X plid=0x%.8X involving "
+                "TPM with HUID=0x%08X, committing Unrecoverable Error "
+                "eid=0x%.8X with same plid=0x%.8X",
+                original_eid, original_plid, TARGETING::get_huid(i_pTpm),
+                pError->eid(), pError->plid());
+
             // Hardware/Procedure callouts/trace should have been added to the
             // original log but the main HW/SW callouts/traces are replicated here
             // just in case.
@@ -1852,7 +1860,7 @@ errlHndl_t tpmReadAttributes ( TARGETING::Target * i_target,
 
     // Printing mux info separately, if combined, nothing is displayed
     char* l_muxPath = io_tpmInfo.i2cMuxPath.toString();
-    TRACFCOMP(g_trac_tpmdd, "tpmReadAttributes(): "
+    TRACUCOMP(g_trac_tpmdd, "tpmReadAttributes(): "
               "muxSelector=0x%X, muxPath=%s",
               io_tpmInfo.i2cMuxBusSelector,
               l_muxPath);
diff --git a/src/usr/secureboot/node_comm/node_comm_dd.C b/src/usr/secureboot/node_comm/node_comm_dd.C
index aa4d2fc1d..44f22bc5f 100644
--- a/src/usr/secureboot/node_comm/node_comm_dd.C
+++ b/src/usr/secureboot/node_comm/node_comm_dd.C
@@ -108,8 +108,32 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
     do
     {
 
+    // Verify OP type
+    if ( (i_opType != DeviceFW::READ) &&
+         (i_opType != DeviceFW::WRITE) )
+    {
+        TRACFCOMP( g_trac_nc,ERR_MRK"nodeCommPerformOp: Invalid opType: 0x%X",
+                   i_opType);
+        /*@
+         * @errortype
+         * @moduleid     MOD_NCDD_PERFORM_OP
+         * @reasoncode   RC_NCDD_INVALID_OP_TYPE
+         * @userdata1    Operation type
+         * @userdata2    Input Target HUID
+         * @devdesc      NodeComm DD invalid operation type
+         * @custdesc     Trusted Boot failure
+         */
+        err = new ERRORLOG::ErrlEntry(
+                            ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                            MOD_NCDD_PERFORM_OP,
+                            RC_NCDD_INVALID_OP_TYPE,
+                            i_opType,
+                            node_comm_args.tgt_huid,
+                            ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+        break;
+    }
+
     // Check other input parameters
-    // @TODO RTC 203642 add check that i_opType is only READ or WRITE
     const auto max_linkId = (mode==NCDD_MODE_ABUS)
                               ? NCDD_MAX_ABUS_LINK_ID
                               : NCDD_MAX_XBUS_LINK_ID;
@@ -149,7 +173,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
                                          node_comm_args.mboxId),
                                        reinterpret_cast<uint64_t>(
                                          node_comm_args.data_ptr),
-                                       true /*Add HB SW Callout*/ );
+                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
         break;
     }
 
diff --git a/src/usr/secureboot/node_comm/node_comm_exchange.C b/src/usr/secureboot/node_comm/node_comm_exchange.C
index fde09af52..ff8ff8a31 100644
--- a/src/usr/secureboot/node_comm/node_comm_exchange.C
+++ b/src/usr/secureboot/node_comm/node_comm_exchange.C
@@ -134,49 +134,61 @@ errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
     errlHndl_t err = nullptr;
     o_nonce = NODE_COMM_DEFAULT_NONCE;
     Target* tpm_tgt = nullptr;
-    TargetHandleList tpmTargetList;
 
     TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusGetRandom:");
 
     do
     {
 
-    // Get all possible functional TPMs
-    TRUSTEDBOOT::getTPMs(tpmTargetList);
+    // Can only use the functional Primary TPM
+    // This function call requires the CONFIG check for compilation purposes,
+    // but no extra error handling is needed as it should not have gotten this
+    // far if CONFIG_TPMDD wasn't set
+#ifdef CONFIG_TPMDD
+    TRUSTEDBOOT::getPrimaryTpm(tpm_tgt);
+#endif
+    HwasState hwasState{};
+    if(tpm_tgt)
+    {
+        hwasState = tpm_tgt->getAttr<TARGETING::ATTR_HWAS_STATE>();
+        TRACUCOMP(g_trac_nc,INFO_MRK
+                  "TPM HUID 0x%08X has state of {present=%d, "
+                  "functional=%d}",
+                  get_huid(tpm_tgt),
+                  hwasState.present,hwasState.functional);
 
-    if (tpmTargetList.size() == 0)
+    }
+
+    if ((tpm_tgt == nullptr) ||
+        (hwasState.functional == false))
     {
         TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusGetRandom: no functional "
-                  "TPMs found - tpmTargetList.size() = %d - Committing "
-                  "predictive error. Continuing using default nonce=0x%.16llX",
-                  tpmTargetList.size(), o_nonce);
+                  "Primary TPM: huid=0x%.08X: functional=%d",
+                  get_huid(tpm_tgt), hwasState.functional);
 
         /*@
          * @errortype
-         * @reasoncode       RC_NCEX_NO_FUNCTIONAL_TPMS
+         * @reasoncode       RC_NCEX_NO_FUNCTIONAL_PRIMARY_TPM
          * @moduleid         MOD_NCEX_GET_RANDOM
-         * @userdata1        <Unused>
-         * @userdata2        <Unused>
-         * @devdesc          No functional TPMs were found
+         * @userdata1        TPM Target HUID
+         * @userdata2[0:31]  TPM Target HWAS State Present
+         * @userdata2[31:63] TPM Target HWAS State Functional
+         * @devdesc          Functional Primary TPM was not found
          * @custdesc         Trusted Boot failure
          */
         err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_PREDICTIVE,
                                        MOD_NCEX_GET_RANDOM,
-                                       RC_NCEX_NO_FUNCTIONAL_TPMS,
-                                       0,
-                                       0,
-                                       true /*Add HB SW Callout*/ );
-
-        // err commited outside of do-while loop below
+                                       RC_NCEX_NO_FUNCTIONAL_PRIMARY_TPM,
+                                       get_huid(tpm_tgt),
+                                       TWO_UINT32_TO_UINT64(
+                                         hwasState.present,
+                                         hwasState.functional),
+                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
 
         // break here to skip calling GetRandom() below
         break;
     }
 
-    // Use first of functional TPM target list
-    // @TODO RTC 203642 Update this to use Primary TPM
-    tpm_tgt = tpmTargetList[0];
-
     // This function call requires the CONFIG check for compilation purposes,
     // but no extra error handling is needed as it should not have gotten this
     // far if CONFIG_TPMDD wasn't set
@@ -206,10 +218,8 @@ errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
 
     if (err)
     {
-        err->collectTrace(SECURE_COMP_NAME);
-        err->collectTrace(NODECOMM_TRACE_NAME);
         err->collectTrace(TRBOOT_COMP_NAME);
-        errlCommit(err, SECURE_COMP_ID);
+        err->collectTrace(NODECOMM_TRACE_NAME);
     }
 
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusGetRandom: "
@@ -591,6 +601,10 @@ errlHndl_t nodeCommGenSlaveQuoteResponse(const MasterQuoteRequestBlob* const i_r
     {
         // There was some error; allocate the output buffer just big enough
         // for an eye catcher and node ID
+        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: An error "
+                 "occurred during slave quote composition. Sending NDNOTPM_ "
+                 "back Master Node after poisoning all TPMs on this node");
+
         NCEyeCatcher_t l_badEyeCatcher = NDNOTPM_;
         o_resp = new uint8_t[sizeof(l_badEyeCatcher) + sizeof(l_nodeId)]{};
         memcpy(o_resp, &l_badEyeCatcher, sizeof(l_badEyeCatcher));
@@ -888,10 +902,10 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
 
         TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                   "my: linkId=%d, mboxId=%d, ObusInstance=%d. "
-                  "expected peer: linkId=%d, mboxId=%d, ObusInstance=%d.",
+                  "expected peer: n%d linkId=%d, mboxId=%d, ObusInstance=%d",
                   my_linkId, my_mboxId, l_obus.myObusInstance,
-                  expected_peer_linkId, expected_peer_mboxId,
-                  l_obus.peerObusInstance);
+                  l_obus.peerNodeInstance, expected_peer_linkId,
+                  expected_peer_mboxId, l_obus.peerObusInstance);
 
         // Get random number from TPM
         msg_format_t msg_data;
@@ -916,6 +930,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferSend(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   l_obus.peerNodeInstance,
                                    NCT_TRANSFER_SBID,
                                    reinterpret_cast<uint8_t*>
                                      (&(msg_data.value)),
@@ -924,7 +939,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         if (err)
         {
             TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 1: "
-                      "nodeCommAbusTransferSend returned an error");
+                      "nodeCommTransferSend returned an error");
             break;
         }
 
@@ -949,6 +964,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   l_obus.peerNodeInstance,
                                    NCT_TRANSFER_SBID,
                                    data_rcv_buffer,
                                    data_rcv_size);
@@ -963,7 +979,6 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         assert(data_rcv_buffer!=nullptr,"nodeCommAbusExchangeMaster: Loop 1: data_rcv_buffer returned as nullptr");
 
         // Add receiver Link Id to the message data
-        // @TODO RTC 203642 Check that data_rcv_size == sizeof(uint64_t)
         // here and in other places where SBID is handled
         memcpy(&(msg_data.value), data_rcv_buffer, data_rcv_size);
         msg_data.receiver_linkId = my_linkId;
@@ -1033,6 +1048,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferSend(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   l_obus.peerNodeInstance,
                                    NCT_TRANSFER_QUOTE_REQUEST,
                                    reinterpret_cast<uint8_t*>
                                      (&quote_request),
@@ -1041,7 +1057,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         if (err)
         {
             TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 2: "
-                      "nodeCommAbusTransferSend returned an error");
+                      "nodeCommTransferSend returned an error");
             break;
         }
 
@@ -1055,6 +1071,7 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   l_obus.peerNodeInstance,
                                    NCT_TRANSFER_QUOTE_RESPONSE,
                                    data_rcv_buffer,
                                    data_rcv_size);
@@ -1155,6 +1172,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   i_obus_instance.peerNodeInstance,
                                    NCT_TRANSFER_SBID,
                                    data_buffer,
                                    data_size);
@@ -1221,6 +1239,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferSend(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   i_obus_instance.peerNodeInstance,
                                    NCT_TRANSFER_SBID,
                                    reinterpret_cast<uint8_t*>
                                      (&(msg_data.value)),
@@ -1254,6 +1273,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   i_obus_instance.peerNodeInstance,
                                    NCT_TRANSFER_QUOTE_REQUEST,
                                    data_buffer,
                                    data_size);
@@ -1267,8 +1287,6 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         // sanity check here to be certain
         assert(data_buffer!=nullptr,"nodeCommAbusExchangeSlave: data_buffer returned as nullptr");
 
-        // @TODO 203642 check that size back is size of MasterQuoteRequestBlob
-
         // Cast the data received into a MasterQuoteRequestBlob
         MasterQuoteRequestBlob quote_request{};
         memcpy(&quote_request, data_buffer, data_size);
@@ -1297,6 +1315,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         err = nodeCommTransferSend(i_mProcInfo.tgt,
                                    my_linkId,
                                    my_mboxId,
+                                   i_obus_instance.peerNodeInstance,
                                    NCT_TRANSFER_QUOTE_RESPONSE,
                                    data_buffer,
                                    data_size);
@@ -1304,7 +1323,7 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         if (err)
         {
             TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
-                      "nodeCommAbusTransferSend returned an error");
+                      "nodeCommTransferSend returned an error");
             break;
         }
 
@@ -1431,7 +1450,7 @@ errlHndl_t nodeCommAbusExchange(void)
                                        RC_NCEX_INVALID_PHYS_PATH,
                                        get_huid(mProcInfo.tgt),
                                        0,
-                                       true /*Add HB SW Callout*/ );
+                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
 
         ERRORLOG::ErrlUserDetailsStringSet path;
         path.add("mProc PHYS Entity Path", l_phys_path_str);
@@ -1654,7 +1673,7 @@ errlHndl_t nodeCommAbusExchange(void)
                                        TWO_UINT32_TO_UINT64(
                                          obus_instances.size(),
                                          total_nodes),
-                                       true /*Add HB SW Callout*/ );
+                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
 
         break;
     }
diff --git a/src/usr/secureboot/node_comm/node_comm_transfer.C b/src/usr/secureboot/node_comm/node_comm_transfer.C
index daa6b6cbb..b7afb02ef 100644
--- a/src/usr/secureboot/node_comm/node_comm_transfer.C
+++ b/src/usr/secureboot/node_comm/node_comm_transfer.C
@@ -31,7 +31,9 @@
 #include <devicefw/userif.H>
 #include <trace/interface.H>
 #include <scom/centaurScomCache.H> // for TRACE_ERR_FMT, TRACE_ERR_ARGS
+#include <targeting/targplatutil.H>
 #include <secureboot/nodecommif.H>
+#include <secureboot/secure_reasoncodes.H>
 #include "node_comm.H"
 #include "node_comm_transfer.H"
 
@@ -50,6 +52,7 @@ namespace NODECOMM
 errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
                                 const uint8_t i_linkId,
                                 const uint8_t i_mboxId,
+                                const uint8_t i_recvNode,
                                 node_comm_transfer_types_t i_transferType,
                                 const uint8_t * i_data,
                                 const size_t i_dataSize)
@@ -63,13 +66,51 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
     size_t total_data_msgs = (i_dataSize + (sizeof(uint64_t)-1))
                              /sizeof(uint64_t);
 
+    auto my_node = TARGETING::UTIL::getCurrentNodePhysId();
+
     TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommTransferSend: iProc=0x%.08X "
               "to send %d bytes of data through linkId=%d mboxId=%d over "
-              "%d data messages",
-              get_huid(i_pProc), i_dataSize, i_linkId, i_mboxId, total_data_msgs);
+              "%d data messages to node %d",
+              get_huid(i_pProc), i_dataSize, i_linkId, i_mboxId,
+              total_data_msgs, i_recvNode);
 
     do
     {
+        // Check expected size of msgType here
+        if ((TransferSizeMap.count(i_transferType) > 0) &&
+            (TransferSizeMap.at(i_transferType) != i_dataSize))
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferSend: iProc=0x%.08X: "
+              "i_dataSize %d bytes does not align with i_transferType 0x%X: "
+              "Expected size %d bytes.",
+              get_huid(i_pProc), i_dataSize, i_transferType,
+              TransferSizeMap.at(i_transferType));
+
+            /*@
+             * @errortype
+             * @reasoncode       RC_NCT_TYPE_SIZE_MISMATCH
+             * @moduleid         MOD_NCT_SEND
+             * @userdata1[0:31]  Input TARGET HUID
+             * @userdata1[32:63] Input Data Size
+             * @userdata2[0:31]  Input Transfer Type
+             * @userdata2[32:63] Expected Size For Input Transfer Type
+             * @devdesc          Invalid Input Args for Node Comm Transfer Send
+             * @custdesc         Trusted Boot failure
+             */
+            err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                          MOD_NCT_SEND,
+                                          RC_NCT_TYPE_SIZE_MISMATCH,
+                                          TWO_UINT32_TO_UINT64(
+                                            get_huid(i_pProc),
+                                            i_dataSize),
+                                          TWO_UINT32_TO_UINT64(
+                                            i_transferType,
+                                            TransferSizeMap.at(
+                                              i_transferType)),
+                                          ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+            break;
+        }
+
         // Keep track of data sent
         size_t bytes_sent = 0;
         size_t bytes_left = i_dataSize;
@@ -89,9 +130,8 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
                 node_comm_msg_format_t send_msg;
                 send_msg.value = 0;  // Clear out the data
 
-                // @TODO RTC 203642 Fix sendingNode and recving Node
-                send_msg.sendingNode = 0;
-                send_msg.recvingNode = 1;
+                send_msg.sendingNode = my_node;
+                send_msg.recvingNode = i_recvNode;
 
                 send_msg.msgType = i_transferType;
                 send_msg.msgSeqNum = msg_seq;
@@ -160,12 +200,60 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
                       "msg %d: receiving ACK = 0x%.16llX",
                       msg_seq, data_recv);
 
+            // Verify that ACK returned has expected data
+            node_comm_msg_format_t ack_msg = {.value = data_recv};
+            auto ack_expected_msg_type = NCT_ACK_OF_DATA;
+            if (msg_seq==0)
+            {
+                ack_expected_msg_type = NCT_ACK_OF_INTIATION;
+            }
 
-
-            // @TODO RTC 203642 Check that ACK is
-            // -- from the right node
-            // -- from right linkId/mboxId
-            // -- the right type of ACK
+            if ((ack_msg.sendingNode != i_recvNode) ||
+                (ack_msg.recvingNode != my_node) ||
+                (ack_msg.msgType     != ack_expected_msg_type) ||
+                (ack_msg.msgSeqNum   != msg_seq))
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferSend: "
+                  "ACK has bad data! Got (expected): sN=%d (%d), rN=%d (%d), "
+                  "msgType=0x%X (0x%X), msgSeqNum=0x%X (0x%X)",
+                  ack_msg.sendingNode, i_recvNode,
+                  ack_msg.recvingNode, my_node,
+                  ack_msg.msgType, ack_expected_msg_type,
+                  ack_msg.msgSeqNum, msg_seq);
+
+                /*@
+                 * @errortype
+                 * @reasoncode       RC_NCT_ACK_MISMATCH
+                 * @moduleid         MOD_NCT_SEND
+                 * @userdata1[0:15]  Actual Node Sending the ACK
+                 * @userdata1[16:31] Expected Node Sending the ACK
+                 * @userdata1[32:47] Actual Node Receiving the ACK
+                 * @userdata1[48:63] Expected Node Receiving the ACK
+                 * @userdata2[0:15]  Actual Message Type from the ACK
+                 * @userdata2[16:31] Expected Message Type from the ACK
+                 * @userdata2[32:47] Actual Sequence Number from the ACK
+                 * @userdata2[48:63] Expected Sequence Number from the ACK
+                 * @devdesc          Invalid data from ACK for Node Comm
+                 *                   Transfer Send
+                 * @custdesc         Trusted Boot failure
+                 */
+                err = new ERRORLOG::ErrlEntry(
+                              ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                              MOD_NCT_SEND,
+                              RC_NCT_ACK_MISMATCH,
+                              FOUR_UINT16_TO_UINT64(
+                                ack_msg.sendingNode,
+                                i_recvNode,
+                                ack_msg.recvingNode,
+                                my_node),
+                              FOUR_UINT16_TO_UINT64(
+                                ack_msg.msgType,
+                                ack_expected_msg_type,
+                                ack_msg.msgSeqNum,
+                                msg_seq),
+                              ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+                break;
+            }
 
         }  // end of for-loop of messages
 
@@ -189,15 +277,20 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
 errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
                                 const uint8_t i_linkId,
                                 const uint8_t i_mboxId,
+                                const uint8_t i_sentNode,
                                 node_comm_transfer_types_t i_transferType,
                                 uint8_t*& o_data,
                                 size_t & o_dataSize)
 {
     errlHndl_t err = nullptr;
 
+    auto my_node = TARGETING::UTIL::getCurrentNodePhysId();
+
     TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommTransferRecv: i_pProc=0x%.08X "
-              "expecting messages of type 0x%.02x from linkId=%d mboxId=%d",
-              get_huid(i_pProc), i_transferType, i_linkId, i_mboxId);
+              "expecting messages of type 0x%.02x from linkId=%d mboxId=%d, "
+              "node=%d",
+              get_huid(i_pProc), i_transferType, i_linkId, i_mboxId,
+              i_sentNode);
 
     // Clear the output variables to be safe
     o_data = nullptr;
@@ -231,17 +324,63 @@ errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
                   init_msg.msgType, init_msg.msgSeqNum, init_msg.totalDataMsgs,
                   init_msg.totalDataSize);
 
-        // @TODO RTC 203642 Add the following checks:
-        // - check nodes
-        // - init_msg.msgSeqNum should be 0 for initiation message
-        // - check that expected transfer type is received
-
-
+        // Verify Data in the Initiation message
+        auto expected_msg_seq = 0;
+        if ((init_msg.sendingNode != i_sentNode) ||
+            (init_msg.recvingNode != my_node) ||
+            (init_msg.msgType     != i_transferType) ||
+            (init_msg.msgSeqNum   != expected_msg_seq))
+        {
+            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferRecv: "
+              "Initiation Message has bad data! Got (expected): "
+              "sN=%d (%d), rN=%d (%d), "
+              "msgType=0x%X (0x%X), msgSeqNum=0x%X (0x%X)",
+              init_msg.sendingNode, i_sentNode,
+              init_msg.recvingNode, my_node,
+              init_msg.msgType, i_transferType,
+              init_msg.msgSeqNum, expected_msg_seq);
+
+            /*@
+             * @errortype
+             * @reasoncode       RC_NCT_INITIATION_MISMATCH
+             * @moduleid         MOD_NCT_RECEIVE
+             * @userdata1[0:15]  Actual Node Sending the Initiation Message
+             * @userdata1[16:31] Expected Node Sending the Initiation Message
+             * @userdata1[32:47] Actual Node Receiving the Initiation Message
+             * @userdata1[48:63] Expected Node Receiving the Initiaion Message
+             * @userdata2[0:15]  Actual Message Type from Initiation Message
+             * @userdata2[16:31] Expected Message Type from Initiation Message
+             * @userdata2[32:47] Actual Sequence Number from Initiation Message
+             * @userdata2[48:63] Expected Sequence Number from Initiation Msg
+             * @devdesc          Invalid data from Initiation Message
+             * @custdesc         Trusted Boot failure
+             */
+            err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                          MOD_NCT_RECEIVE,
+                                          RC_NCT_INITIATION_MISMATCH,
+                                          FOUR_UINT16_TO_UINT64(
+                                            init_msg.sendingNode,
+                                            i_sentNode,
+                                            init_msg.recvingNode,
+                                            my_node),
+                                          FOUR_UINT16_TO_UINT64(
+                                            init_msg.msgType,
+                                            i_transferType,
+                                            init_msg.msgSeqNum,
+                                            expected_msg_seq),
+                                          ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+            break;
+        }
         // Reply with ACK:
         node_comm_msg_format_t ack_msg = {.value = 0};
         ack_msg.value = 0;
 
-        // @TODO RTC 203642 Fix sending node Ids
+        // Set node info
+        // NOTE: These values are from the perspective of
+        // sending the ACK and *not* from sending/receiving the data
+        ack_msg.sendingNode = TARGETING::UTIL::getCurrentNodePhysId();
+        ack_msg.recvingNode = i_sentNode;
+
         // Always set these for ACK of Initiation:
         ack_msg.msgSeqNum = 0;
         ack_msg.msgType = NCT_ACK_OF_INTIATION;
@@ -318,7 +457,7 @@ errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
             bytes_read += loop_bytes_read;
             bytes_left -= loop_bytes_read;
 
-            // Send Data ACK
+            // Send Data ACK (re-uses previous ack_msg settings)
             ack_msg.msgType = NCT_ACK_OF_DATA;
             ack_msg.msgSeqNum = msg_seq;
             ack_msg.totalDataSize = bytes_read;
@@ -356,9 +495,49 @@ errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
         {
             o_data = data_buffer;
             o_dataSize = init_msg.totalDataSize;
-        }
 
 
+            // Check size of data returned against the expected amount based
+            // on the msgType
+            if ((TransferSizeMap.count(i_transferType) > 0) &&
+                (TransferSizeMap.at(i_transferType) != o_dataSize))
+            {
+                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommTransferReceive: "
+                          "iProc=0x%.08X: o_dataSize %d bytes does not "
+                          "align with i_transferType 0x%X: "
+                          "Expected size %d bytes.",
+                          get_huid(i_pProc), o_dataSize, i_transferType,
+                          TransferSizeMap.at(i_transferType));
+
+                /*@
+                 * @errortype
+                 * @reasoncode       RC_NCT_TYPE_SIZE_MISMATCH
+                 * @moduleid         MOD_NCT_RECEIVE
+                 * @userdata1[0:31]  Input TARGET HUID
+                 * @userdata1[32:63] Output Data Size
+                 * @userdata2[0:31]  Input Transfer Type
+                 * @userdata2[32:63] Expected Size For Input Transfer Type
+                 * @devdesc          Unexpected Size of Data Received based on
+                 *                   Input Transfer Type
+                 * @custdesc         Trusted Boot failure
+                 */
+                err = new ERRORLOG::ErrlEntry(
+                              ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                              MOD_NCT_RECEIVE,
+                              RC_NCT_TYPE_SIZE_MISMATCH,
+                              TWO_UINT32_TO_UINT64(
+                                get_huid(i_pProc),
+                                o_dataSize),
+                              TWO_UINT32_TO_UINT64(
+                                i_transferType,
+                                TransferSizeMap.at(
+                                  i_transferType)),
+                              ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+                break;
+            }
+
+        }
+
     } while( 0 );
 
     TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommTransferRecv: "
diff --git a/src/usr/secureboot/node_comm/node_comm_transfer.H b/src/usr/secureboot/node_comm/node_comm_transfer.H
index f64b1e387..201661447 100644
--- a/src/usr/secureboot/node_comm/node_comm_transfer.H
+++ b/src/usr/secureboot/node_comm/node_comm_transfer.H
@@ -30,6 +30,7 @@
 // ----------------------------------------------
 #include <config.h>
 #include "node_comm.H"
+#include <map>
 
 // ----------------------------------------------
 // Defines
@@ -56,7 +57,6 @@ enum node_comm_transfer_types_t : uint8_t
     // Types of ACKs (Ackowledgments)
     NCT_ACK_OF_INTIATION           =  0x01,
     NCT_ACK_OF_DATA                =  0x02,
-    // @TODO RTC 203642: ACK of non-support and shutting down the channel?
 
     // Message Transfer Types
     NCT_TRANSFER_SBID              =  0x10,
@@ -85,6 +85,15 @@ union node_comm_msg_format_t
     } PACKED;
 };
 
+/**
+ * @brief Map of expected sizes associated with Message Transfer Types
+ */
+const std::map<node_comm_transfer_types_t, size_t>
+    TransferSizeMap = {{NCT_TRANSFER_SBID,
+                            sizeof(uint64_t)},
+                         {NCT_TRANSFER_QUOTE_REQUEST,
+                            sizeof(MasterQuoteRequestBlob)}};
+
 
 /**
  *  @brief This function sends multiple messages with data over the ABUS from
@@ -94,10 +103,14 @@ union node_comm_msg_format_t
  *                               Can't be nullptr
  *  @param[in] i_linkId        - Link Id Message is sent from
  *  @param[in] i_mboxId        - Mailbox Id Message is sent from
+ *  @param[in] i_recvNode      - Node expected to receive the message
  *  @param[in] i_transferType  - Transfer Message Type to send
  *  @param[in] i_data          - Data to be sent
  *  @param[in] i_dataSize      - Size of Data to be sent in bytes
  *
+ *  @note i_dataSize is verified to match i_transferType when applicable
+ *        based on TransferSizeMap.
+ *
  *  @return errlHndl_t Error log handle
  *  @retval nullptr Operation was successful
  *  @retval !nullptr Operation failed with valid error log
@@ -105,6 +118,7 @@ union node_comm_msg_format_t
 errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
                                 uint8_t i_linkId,
                                 uint8_t i_mboxId,
+                                uint8_t i_recvNode,
                                 node_comm_transfer_types_t i_transferType,
                                 const uint8_t * i_data,
                                 size_t i_dataSize);
@@ -117,6 +131,7 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
  *                                Can't be nullptr
  *  @param[in]  i_linkId        - Link Id Message is expected from
  *  @param[in]  i_mboxId        - Mailbox Id Message is expected from
+ *  @param[in]  i_sentNode      - Node expected to have sent the message
  *  @param[in]  i_transferType  - Expected Transfer Message Type to receive
  *                                Failure if different type is received
  *  @param[out] o_data          - Data received
@@ -127,6 +142,9 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
  *          If an error is returned, o_data will be nullptr and o_dataSize
  *          will be zero.
  *
+ *  @note o_dataSize is verified to match i_transferType when applicable
+ *        based on TransferSizeMap.
+ *
  *  @return errlHndl_t Error log handle
  *  @retval nullptr Operation was successful
  *  @retval !nullptr Operation failed with valid error log
@@ -134,6 +152,7 @@ errlHndl_t nodeCommTransferSend(TARGETING::Target* i_pProc,
 errlHndl_t nodeCommTransferRecv(TARGETING::Target* i_pProc,
                                 uint8_t i_linkId,
                                 uint8_t i_mboxId,
+                                uint8_t i_sentNode,
                                 node_comm_transfer_types_t i_transferType,
                                 uint8_t*& o_data,
                                 size_t & o_dataSize);
diff --git a/src/usr/secureboot/trusted/trustedboot.C b/src/usr/secureboot/trusted/trustedboot.C
index 9c075dc7d..66c3a8664 100644
--- a/src/usr/secureboot/trusted/trustedboot.C
+++ b/src/usr/secureboot/trusted/trustedboot.C
@@ -1330,6 +1330,12 @@ void tpmVerifyFunctionalPrimaryTpmExists(
                                               MOD_TPM_VERIFYFUNCTIONAL,
                                               RC_TPM_NOFUNCTIONALTPM_FAIL);
 
+                TRACFCOMP(g_trac_trustedboot, ERR_MRK
+                          "tpmVerifyFunctionalPrimaryTpmExists: Shutting down "
+                          "system because no Functional Primary TPM was found "
+                          "but system policy required it. errl EID 0x%08X",
+                          err->eid());
+
                 // Add low priority HB SW callout
                 err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
                                          HWAS::SRCI_PRIORITY_LOW);
@@ -1362,8 +1368,9 @@ void tpmVerifyFunctionalPrimaryTpmExists(
                     err = TARGETING::AttrRP::syncAllAttributesToFsp();
                     if(err)
                     {
-                        TRACFCOMP(g_trac_trustedboot, ERR_MRK"Could not sync"
-                                  " attributes to FSP; errl EID 0x%08X",
+                        TRACFCOMP(g_trac_trustedboot, ERR_MRK
+                                  "tpmVerifyFunctionalPrimaryTpmExists: Could "
+                                  "not sync attributes to FSP; errl EID 0x%08X",
                                   err->eid());
                         errlCommit(err, TRBOOT_COMP_ID);
                     }
@@ -1376,13 +1383,13 @@ void tpmVerifyFunctionalPrimaryTpmExists(
             else
             {
                 TRACUCOMP(g_trac_trustedboot,
-                          "No functional primary TPM found but"
-                                                            "TPM not Required");
+                          "tpmVerifyFunctionalPrimaryTpmExists: No functional "
+                          "primary TPM found but TPM not Required");
             }
         }
         else
         {
-            TRACUCOMP(g_trac_trustedboot,
+            TRACUCOMP(g_trac_trustedboot,"tpmVerifyFunctionalPrimaryTpmExists: "
                       "No functional primary TPM found but not running secure");
         }
 
@@ -2117,6 +2124,11 @@ bool getTpmRequiredSensorValue(bool& o_isTpmRequired)
                    "not available: retVal=%d (sensorNum=0x%X)",
                    retVal, sensorNum );
     }
+
+    TRACFCOMP( g_trac_trustedboot,
+               "getTpmRequiredSensorValue: isAvail=%s, o_isTpmRequired=%s",
+               (retVal ? "Yes" : "No"),
+               (o_isTpmRequired ? "Yes" : "No") );
 #else
     // IPMI support not there, so consider sensor not available
     retVal = false;
@@ -2125,12 +2137,6 @@ bool getTpmRequiredSensorValue(bool& o_isTpmRequired)
                retVal );
 #endif
 
-
-    TRACFCOMP( g_trac_trustedboot,
-               "getTpmRequiredSensorValue: isAvail=%s, o_isTpmRequired=%s",
-               (retVal ? "Yes" : "No"),
-               (o_isTpmRequired ? "Yes" : "No") );
-
     return retVal;
 }