path: root/src/usr/secureboot/node_comm/node_comm_exchange.C

/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/* $Source: src/usr/secureboot/node_comm/node_comm_exchange.C $           */
/*                                                                        */
/* OpenPOWER HostBoot Project                                             */
/*                                                                        */
/* Contributors Listed Below - COPYRIGHT 2018,2019                        */
/* [+] International Business Machines Corp.                              */
/*                                                                        */
/*                                                                        */
/* Licensed under the Apache License, Version 2.0 (the "License");        */
/* you may not use this file except in compliance with the License.       */
/* You may obtain a copy of the License at                                */
/*                                                                        */
/*     http://www.apache.org/licenses/LICENSE-2.0                         */
/*                                                                        */
/* Unless required by applicable law or agreed to in writing, software    */
/* distributed under the License is distributed on an "AS IS" BASIS,      */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
/* implied. See the License for the specific language governing           */
/* permissions and limitations under the License.                         */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */
/**
 * @file node_comm_exchange.C
 *
 * @brief Runs the procedure for the drawers/nodes to exchange messages
 *        over the ABUS Link Mailbox facility
 *
 */

// ----------------------------------------------
// Includes
// ----------------------------------------------
#include <string.h>
#include <algorithm>
#include <trace/interface.H>
#include <errl/errlentry.H>
#include <errl/errlmanager.H>
#include <errl/errludtarget.H>
#include <errl/errludstring.H>
#include <targeting/common/targetservice.H>
#include <devicefw/userif.H>
#include <devicefw/driverif.H>
#include <secureboot/secure_reasoncodes.H>
#include <secureboot/nodecommif.H>
#include <secureboot/trustedbootif.H>
#include <targeting/common/commontargeting.H>
#include <targeting/common/utilFilter.H>
#include <targeting/targplatutil.H>
#include <sys/internode.h>
#include <util/misc.H>

#include "node_comm.H"
#include "node_comm_transfer.H"

#include <secureboot/service.H>
#include <securerom/contrib/sha512.H>

#include "../trusted/trustedTypes.H"
#include "../trusted/tpmLogMgr.H"

// ----------------------------------------------
// Defines
// ----------------------------------------------
// Use if there is an issue getting random number from a functional TPM
#define NODE_COMM_DEFAULT_NONCE 0xFFFFFFFFFFFFFFFF

using   namespace   TARGETING;

namespace SECUREBOOT
{

namespace NODECOMM
{

/*
 * Struct used to collect data about current node's master proc and other info
 */
struct master_proc_info_t
{
    TARGETING::Target* tgt = nullptr;
    uint8_t     procInstance = 0;
    uint8_t     nodeInstance = 0;
};


/*
 * Struct used to hold data about obus instances
 */
struct obus_instances_t
{
    uint8_t  myObusInstance = 0;
    uint8_t  myObusRelLink  = 0;

    // Expect data to be received on the PEER_PATH instances
    uint8_t  peerNodeInstance = 0;
    uint8_t  peerProcInstance = 0;
    uint8_t  peerObusInstance = 0;
};

/*
 * Union used to hold the 3-tuple of information passed between the nodes -
 *       master node to every sibling node, and all those sibling nodes
 *       back to the master node
 */
union msg_format_t
{
    uint64_t value;
    struct
    {
        uint64_t origin_linkId   : 4;  // relative to the originator
        uint64_t receiver_linkId : 4;  // relative to the receiver
        uint64_t nonce           : 56; // a cryptographically strong random
                                       // nummber requested from the TPM
    } PACKED;
};

/**
 *  @brief This function generates a nonce by calling GetRandom on an
 *         available TPM.
 *
 *  @param[out] o_nonce - The nonce that is generated
 *  @note       Nonce is only valid if the operation is successful
 *
 *  @return errlHndl_t Error log handle
 *  @retval nullptr Operation was successful
 *  @retval !nullptr Operation failed with valid error log
 */
errlHndl_t nodeCommAbusGetRandom(uint64_t & o_nonce)
{
    errlHndl_t err = nullptr;
    o_nonce = NODE_COMM_DEFAULT_NONCE;
#ifdef CONFIG_TPMDD
    Target* tpm_tgt = nullptr;

    TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusGetRandom:");

    do
    {

    // Can only use the functional Primary TPM
    // This function call requires the CONFIG check for compilation purposes,
    // but no extra error handling is needed as it should not have gotten this
    // far if CONFIG_TPMDD wasn't set
    TRUSTEDBOOT::getPrimaryTpm(tpm_tgt);
    HwasState hwasState{};
    if(tpm_tgt)
    {
        hwasState = tpm_tgt->getAttr<TARGETING::ATTR_HWAS_STATE>();
        TRACUCOMP(g_trac_nc,INFO_MRK
                  "TPM HUID 0x%08X has state of {present=%d, "
                  "functional=%d}",
                  get_huid(tpm_tgt),
                  hwasState.present,hwasState.functional);

    }

    if ((tpm_tgt == nullptr) ||
        (hwasState.functional == false))
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusGetRandom: no functional "
                  "Primary TPM: huid=0x%.08X: functional=%d",
                  get_huid(tpm_tgt), hwasState.functional);

        /*@
         * @errortype
         * @reasoncode       RC_NCEX_NO_FUNCTIONAL_PRIMARY_TPM
         * @moduleid         MOD_NCEX_GET_RANDOM
         * @userdata1        TPM Target HUID
         * @userdata2[0:31]  TPM Target HWAS State Present
         * @userdata2[31:63] TPM Target HWAS State Functional
         * @devdesc          Functional Primary TPM was not found
         * @custdesc         Trusted Boot failure
         */
        err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_PREDICTIVE,
                                       MOD_NCEX_GET_RANDOM,
                                       RC_NCEX_NO_FUNCTIONAL_PRIMARY_TPM,
                                       get_huid(tpm_tgt),
                                       TWO_UINT32_TO_UINT64(
                                         hwasState.present,
                                         hwasState.functional),
                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );

        // break here to skip calling GetRandom() below
        break;
    }

    // This function call requires the CONFIG check for compilation purposes,
    // but no extra error handling is needed as it should not have gotten this
    // far if CONFIG_TPMDD wasn't set
    err = TRUSTEDBOOT::GetRandom(tpm_tgt,
                                 sizeof(o_nonce),
                                 reinterpret_cast<uint8_t*>(&o_nonce));
    if (err)
    {
        // Reset just to make sure above call didn't change it
        o_nonce = NODE_COMM_DEFAULT_NONCE;
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusGetRandom: GetRandom "
                  "returned a fail for TPM 0x%.08X. Commiting err: "
                  TRACE_ERR_FMT
                  ". Using default nonce=0x%.16llX",
                  get_huid(tpm_tgt),
                  TRACE_ERR_ARGS(err),
                  o_nonce);
        // break to be safe in case code gets added later
        break;
    }

    } while( 0 );

    if(err)
    {
        if(!TRUSTEDBOOT::isTpmRequired())
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusGetRandom: Error occurred; "
                      "RC: 0x%.04X; PLID: 0x%.08X. TPM Required policy is off; "
                      "deleting the error and trying to continue.",
                      err->reasonCode(),
                      err->plid());
            // TPM is not required - do not return the error
            delete err;
            err = nullptr;
        }
        else
        {
            err->collectTrace(TRBOOT_COMP_NAME);
            err->collectTrace(NODECOMM_TRACE_NAME);
        }
    }

    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusGetRandom: "
              "nonce=0x%.16llX from TPM=0x%.08X. "
              TRACE_ERR_FMT,
              o_nonce, get_huid(tpm_tgt),
              TRACE_ERR_ARGS(err));

#endif
    return err;

} // end of nodeCommAbusGetRandom

/**
 *  @brief This function logs a nonce to all available TPMs on the node by
 *         extending it to a specific PCR.
 *
 *  @param[in]  i_nonce - The nonce to be logged/extended
 *
 *  @return errlHndl_t Error log handle
 *  @retval nullptr Operation was successful
 *  @retval !nullptr Operation failed with valid error log
 */
errlHndl_t nodeCommAbusLogNonce(uint64_t & i_nonce)
{
    errlHndl_t err = nullptr;

    TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusLogNonce: i_nonce=0x%.16llX",
              i_nonce);

    do
    {

    // Extend the nonce asychronously to all available TPMs
    uint8_t l_digest[sizeof(i_nonce)]={0};
    memcpy(l_digest, &i_nonce, sizeof(i_nonce));

    uint8_t l_logMsg[] = "Node Nonce";
    err = TRUSTEDBOOT::pcrExtend(TRUSTEDBOOT::PCR_1,
                                 TRUSTEDBOOT::EV_PLATFORM_CONFIG_FLAGS,
                                 l_digest,
                                 sizeof(uint64_t),
                                 l_logMsg,
                                 sizeof(l_logMsg));
    if (err)
    {
       TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusLogNonce: pcrExtend "
                  "returned a fail: "
                  TRACE_ERR_FMT,
                  TRACE_ERR_ARGS(err));
    }

    } while( 0 );

    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusLogNonce: i_nonce=0x%.16llX. "
              TRACE_ERR_FMT,
              i_nonce,
              TRACE_ERR_ARGS(err));

    return err;

} // end of nodeCommAbusLogNonce

/**
 * @brief A function to create the slave node quote response that consists of
 *        eye catcher, slave node ID, quote and signature data (represented by
 *        the QuoteDataOut structure), the contents of PCRs 0-7, the
 *        Attestation Key Certificate returned from TPM, the size
 *        and the contents of the TPM log
 * @param[in] i_request the master node request structure
 * @param[out] o_size the size of the slave quote
 * @param[out] o_resp the slave quote in binary format
 * @note Assuming CONFIG_TPMDD is compiled in, o_resp is always allocated
 *       dynamically in this function, and it is the responsibility of the
 *       calller to delete it after the function returns. If an error occurs
 *       within the function, the o_resp will only have an eye catcher
 *       indicating that the slave quote is bad.
 * @return nullptr on success; non-nullptr on error
 */
errlHndl_t nodeCommGenSlaveQuoteResponse(const MasterQuoteRequestBlob* const i_request,
                                         size_t& o_size,
                                         uint8_t*& o_resp)
{
    errlHndl_t l_errl = nullptr;
#ifdef CONFIG_TPMDD
    TRACFCOMP(g_trac_nc, ENTER_MRK"nodeCommGenSlaveQuoteResponse");
    bool l_tpmRequired = TRUSTEDBOOT::isTpmRequired();
    bool l_errorOccurred = false;

    TRUSTEDBOOT::QuoteDataOut l_quoteData;
    uint32_t l_nodeId = TARGETING::UTIL::getCurrentNodePhysId();
    do {

    o_resp = nullptr;

    TARGETING::Target* l_primaryTpm = nullptr;
    TRUSTEDBOOT::getPrimaryTpm(l_primaryTpm);
    if(!l_primaryTpm ||
       !l_primaryTpm->getAttr<TARGETING::ATTR_HWAS_STATE>().functional ||
       !l_primaryTpm->getAttr<TARGETING::ATTR_HWAS_STATE>().present)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: primary TPM not found or not functional");
        l_errorOccurred = true;
        break;
    }


    // If Master indicated that there is an issue with its TPM: Case 1: If the
    // TPM Required policy is on, terminate the boot; Case 2: If TPM required
    // policy is off, send back a token indicating that no nodecomm TPM commands
    // have been performed (remote attestation is not possible with bad master
    // TPM); do not fail the boot.
    if(i_request->EyeCatcher == MSTNOTPM)
    {
        l_errorOccurred = true;
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: Master indicated an issue with secure nodecomm (master eye catcher is MSTNOTPM)");
        // Case 1
        if(l_tpmRequired)
        {
            /* @
             * @errortype
             * @reasoncode RC_NC_BAD_MASTER_TPM
             * @moduleid   MOD_NC_GEN_SLAVE_RESPONSE
             * @userdata1  <unused>
             * @userdata2  <unused>
             * @devdesc    The system policy is set to not allow boot without a
             *             functioning TPM, but the system's master node
             *             indicated that its TPM is compromised. This slave
             *             node must terminate the boot process.
             * @custdesc   Trustedboot failure
             */
             l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                              MOD_NC_GEN_SLAVE_RESPONSE,
                                              RC_NC_BAD_MASTER_TPM);
            // It is unlikely that there is an issue with this node, but collect
            // the logs anyway for ease of debug.
            l_errl->collectTrace(SECURE_COMP_NAME);
            l_errl->collectTrace(TRBOOT_COMP_NAME);
            l_errl->collectTrace(NODECOMM_TRACE_NAME);


            break;
        }
        // Case 2
        // The error condition will be handled below in the if branch that
        // checks l_errorOccurred
    }
    else if(i_request->EyeCatcher != MASTERQ_)
    {
        l_errorOccurred = true;
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: Invalid master eye catcher received: 0x%x", i_request->EyeCatcher);
        if(l_tpmRequired)
        {
            /* @
             * @errortype
             * @reasoncode RC_NC_BAD_MASTER_EYE_CATCH
             * @moduleid   MOD_NC_GEN_SLAVE_RESPONSE
             * @userdata1  Eye catcher received from master
             * @devdesc    Master node sent an unrecognized eye catcher
             * @custdesc   Trustedboot failure
             */
             l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                              MOD_NC_GEN_SLAVE_RESPONSE,
                                              RC_NC_BAD_MASTER_EYE_CATCH,
                                              i_request->EyeCatcher);
            // It is unlikely that there is an issue with this node, but collect
            // the logs anyway for ease of debug.
            l_errl->collectTrace(NODECOMM_TRACE_NAME);
            l_errl->collectTrace(TRBOOT_COMP_NAME);
            l_errl->collectTrace(SECURE_COMP_NAME);
        }
        break;
    }

    // Step 1: Recreate node Attestation Key (AK)
    l_errl = TRUSTEDBOOT::createAttestationKeys(l_primaryTpm);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not create attestation keys");
        break;
    }

    TRUSTEDBOOT::TPM2B_MAX_NV_BUFFER l_AKCert;
    // Step 2: Read the AK Certificate
    l_errl = TRUSTEDBOOT::readAKCertificate(l_primaryTpm, &l_AKCert);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not read AK certificate");
        break;
    }

    // Hash the AK Certificate and extend the hash into PCR1
    SHA512_t l_AKCertHash = {0};
    hashBlob(l_AKCert.buffer,
             sizeof(l_AKCert.buffer),
             l_AKCertHash);

    l_errl = TRUSTEDBOOT::pcrExtend(TRUSTEDBOOT::PCR_1,
                                    TRUSTEDBOOT::EV_PLATFORM_CONFIG_FLAGS,
                                    l_AKCertHash,
                                    sizeof(l_AKCertHash),
                                    l_AKCert.buffer,
                                    sizeof(l_AKCert.buffer));
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not extend AK Certificate hash to TPM");
        break;
    }

    l_quoteData.data = new uint8_t[1 * KILOBYTE]{};// the actual data size will
                                                   // be smaller than 1KB
    // Step 3: Generate quote and signature data (presented as binary data in
    // the QuoteDataOut structure)
    l_errl = TRUSTEDBOOT::generateQuote(l_primaryTpm,
                                        &i_request->MasterNonce,
                                        &l_quoteData);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not generate TPM quote");
        break;
    }

    // Step 4: Flush the AK from the TPM
    l_errl = TRUSTEDBOOT::flushContext(l_primaryTpm);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not flush TPM context");
        break;
    }

    // Step 5: Read the selected PCRs
    // Make sure there is only 1 algo selection
    assert(i_request->PcrSelect.count == 1, "nodeCommGenSlaveQuoteResponse: only 1 hash algo is supported for PCR read");
    uint32_t l_pcrCount = 0;
    TRUSTEDBOOT::TPM_Pcr l_pcrRegs[TRUSTEDBOOT::FW_USED_PCR_COUNT] = {
                                                  TRUSTEDBOOT::PCR_0,
                                                  TRUSTEDBOOT::PCR_1,
                                                  TRUSTEDBOOT::PCR_2,
                                                  TRUSTEDBOOT::PCR_3,
                                                  TRUSTEDBOOT::PCR_4,
                                                  TRUSTEDBOOT::PCR_5,
                                                  TRUSTEDBOOT::PCR_6,
                                                  TRUSTEDBOOT::PCR_7
                                                 };
    TRUSTEDBOOT::TPM_Alg_Id l_algId = static_cast<TRUSTEDBOOT::TPM_Alg_Id>
                            (i_request->PcrSelect.pcrSelections[0].algorithmId);
    size_t l_digestSize = TRUSTEDBOOT::getDigestSize(l_algId);

    // An array of PCR Digest structures to hold the contents of the selected
    // PCRs
    TRUSTEDBOOT::TPM2B_DIGEST l_pcrDigests[TRUSTEDBOOT::FW_USED_PCR_COUNT] {};

    // Iterate through PCRs that hostboot interacts with (PCR0-7) and see
    // if any of those were seleceted; read the selected ones
    for(const auto l_pcr : l_pcrRegs)
    {
        if((i_request->PcrSelect.pcrSelections[0]
            .pcrSelect[l_pcr/TRUSTEDBOOT::FW_USED_PCR_COUNT]) &
            (0x01 << (l_pcr % TRUSTEDBOOT::FW_USED_PCR_COUNT)))
        {
            ++l_pcrCount;
            l_pcrDigests[l_pcr].size = l_digestSize;
            l_errl = TRUSTEDBOOT::pcrRead(l_primaryTpm,
                                      l_pcr,
                                      l_algId,
                                      l_digestSize,
                                      l_pcrDigests[l_pcr].buffer);
            if(l_errl)
            {
                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not read PCR%d", l_pcr);
                break;
            }
        }
    }
    if(l_errl)
    {
        break;
    }

    // Step 6: Read out the primary TPM's Log and copy it into the quote
    TRUSTEDBOOT::TpmLogMgr* l_primaryLogMgr =
                                        TRUSTEDBOOT::getTpmLogMgr(l_primaryTpm);
    if(!l_primaryLogMgr)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: could not fetch primary TPM's log");
        /*@
         * @errortype
         * @reasoncode RC_NC_NO_PRIMARY_TPM_LOG
         * @moduleid   MOD_NC_GEN_SLAVE_RESPONSE
         * @userdata1  Primary TPM HUID
         * @devdesc    Could not fetch primary TPM's Log
         * @custdes    Trustedboot failure
         */
        l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                         MOD_NC_GEN_SLAVE_RESPONSE,
                                         RC_NC_NO_PRIMARY_TPM_LOG,
                                         get_huid(l_primaryTpm));
        l_errl->collectTrace(NODECOMM_TRACE_NAME);
        l_errl->collectTrace(SECURE_COMP_NAME);
        l_errl->collectTrace(TRBOOT_COMP_NAME);
        break;
    }
    uint32_t l_logSize = TRUSTEDBOOT::TpmLogMgr_getLogSize(l_primaryLogMgr);
    const uint8_t* l_logPtr =
                         TRUSTEDBOOT::TpmLogMgr_getLogStartPtr(l_primaryLogMgr);

    NCEyeCatcher_t l_goodEyeCatch = NODEQUOT;

    // Figure out the size of the slave quote
    o_size = sizeof(l_goodEyeCatch) +
             sizeof(l_nodeId) +
             sizeof(l_quoteData.size) +
             l_quoteData.size +
             sizeof(l_pcrCount) +
             // Only include the read PCRs in the slave quote
             sizeof(TRUSTEDBOOT::TPM2B_DIGEST) * l_pcrCount +
             sizeof(l_AKCert) +
             sizeof(l_logSize) +
             l_logSize;

    // Allocate the output
    o_resp = new uint8_t[o_size] {};
    // Now populate the output
    size_t l_currentOffset = 0;

    // First the good eye catcher
    memcpy(o_resp, &l_goodEyeCatch, sizeof(l_goodEyeCatch));
    l_currentOffset += sizeof(l_goodEyeCatch);
    // Now the node ID
    memcpy(o_resp + l_currentOffset, &l_nodeId, sizeof(l_nodeId));
    l_currentOffset += sizeof(l_nodeId);
    // The size of the quote and signature structures
    memcpy(o_resp + l_currentOffset,&l_quoteData.size,sizeof(l_quoteData.size));
    l_currentOffset += sizeof(l_quoteData.size);
    // The TPM quote & signature information (both are included in the TPM
    // quote blob)
    memcpy(o_resp + l_currentOffset, l_quoteData.data, l_quoteData.size);
    l_currentOffset += l_quoteData.size;
    // The number of PCRs read
    memcpy(o_resp + l_currentOffset, &l_pcrCount, sizeof(l_pcrCount));
    l_currentOffset += sizeof(l_pcrCount);
    // PCR contents
    for(const auto l_pcr : l_pcrRegs)
    {
        if(l_pcrDigests[l_pcr].size != 0)
        {
            // Copy the size of the PCR
            memcpy(o_resp + l_currentOffset,
                   &l_pcrDigests[l_pcr].size,
                   sizeof(l_pcrDigests[l_pcr].size));
            l_currentOffset += sizeof(l_pcrDigests[l_pcr].size);
            // Now the actual data
            memcpy(o_resp + l_currentOffset,
                   l_pcrDigests[l_pcr].buffer,
                   l_pcrDigests[l_pcr].size);
            l_currentOffset += l_pcrDigests[l_pcr].size;
        }
    }
    // AK certificate size
    memcpy(o_resp + l_currentOffset, &l_AKCert.size, sizeof(l_AKCert.size));
    l_currentOffset += sizeof(l_AKCert.size);
    // Actual AK certificate
    memcpy(o_resp + l_currentOffset, l_AKCert.buffer, l_AKCert.size);
    l_currentOffset += l_AKCert.size;
    // The length of the TPM log
    memcpy(o_resp + l_currentOffset, &l_logSize, sizeof(l_logSize));
    l_currentOffset += sizeof(l_logSize);
    // The actual TPM log
    memcpy(o_resp + l_currentOffset, l_logPtr, l_logSize);

    } while(0);

    if(l_quoteData.data)
    {
        delete[](l_quoteData.data);
        l_quoteData.data = nullptr;
    }

    if(l_errl || l_errorOccurred)
    {
        // There was some error; allocate the output buffer just big enough
        // for an eye catcher and node ID
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: An error "
                 "occurred during slave quote composition. Sending NDNOTPM_ "
                 "back Master Node after poisoning all TPMs on this node");

        NCEyeCatcher_t l_badEyeCatcher = NDNOTPM_;
        o_resp = new uint8_t[sizeof(l_badEyeCatcher) + sizeof(l_nodeId)]{};
        memcpy(o_resp, &l_badEyeCatcher, sizeof(l_badEyeCatcher));
        memcpy(o_resp + sizeof(l_badEyeCatcher), &l_nodeId, sizeof(l_nodeId));
        o_size = sizeof(l_badEyeCatcher) + sizeof(l_nodeId);

        errlHndl_t l_poisonTpmErr = TRUSTEDBOOT::poisonAllTpms();
        if(l_poisonTpmErr)
        {
            if(l_errl)
            {
                l_poisonTpmErr->plid(l_errl->plid());
            }
            if(l_tpmRequired)
            {
                errlCommit(l_poisonTpmErr, SECURE_COMP_ID);
            }
            else
            {
                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenSlaveQuoteResponse: "
                          "Could not poison TPMs. Errl PLID: 0x%.08X "
                          "Deleting the error log and continuing anyway.",
                          l_poisonTpmErr->plid());
                delete l_poisonTpmErr;
                l_poisonTpmErr = nullptr;
            }
        }
    }

    TRACFCOMP(g_trac_nc, EXIT_MRK"nodeCommGenSlaveQuoteResponse: " TRACE_ERR_FMT, TRACE_ERR_ARGS(l_errl));
#endif
    return l_errl;
} //nodeCommGenSlaveQuoteResponse

/**
 * @brief A function to generate a master quote request blob that will be sent
 *        to the slave node(s) as part of the node communication protocol. The
 *        master request consists of an eye catcher, 32-byte TPM-generated
 *        random number, and a PCR selection structure.
 * @param[out] o_request the output master quote request data structure
 * @return nullptr on success; non-nullptr on error
 */
errlHndl_t nodeCommGenMasterQuoteRequest(MasterQuoteRequestBlob* const o_request)
{
    errlHndl_t l_errl = nullptr;
#ifdef CONFIG_TPMDD
    TRACFCOMP(g_trac_nc, ENTER_MRK"nodeCommGenMasterQuoteRequest");
    bool l_tpmRequired = TRUSTEDBOOT::isTpmRequired();
    do {
    TARGETING::Target* l_primaryTpm = nullptr;
    TRUSTEDBOOT::getPrimaryTpm(l_primaryTpm);
    if(!l_primaryTpm ||
       !l_primaryTpm->getAttr<TARGETING::ATTR_HWAS_STATE>().functional ||
       !l_primaryTpm->getAttr<TARGETING::ATTR_HWAS_STATE>().present)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenMasterQuoteRequest: primary TPM not found or is not functional");

        l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                         MOD_NC_GEN_MASTER_REQUEST,
                                         RC_NC_BAD_MASTER_TPM);
        l_errl->collectTrace(SECURE_COMP_NAME);
        l_errl->collectTrace(TRBOOT_COMP_NAME);
        l_errl->collectTrace(NODECOMM_TRACE_NAME);
        break;
    }

    o_request->EyeCatcher = MASTERQ_;

    // Generate the 32-byte nonce for master request
    l_errl = TRUSTEDBOOT::GetRandom(l_primaryTpm,
                                    sizeof(o_request->MasterNonce),
                                    o_request->MasterNonce);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenMasterQuoteRequest: GetRandom failed");
        break;
    }

    // Select PCRs (PCR0-7) to include in slave quote response
    o_request->PcrSelect.count = 1; // One algorithm
    o_request->PcrSelect.pcrSelections[0].algorithmId =
                                                    TRUSTEDBOOT::TPM_ALG_SHA256;
    o_request->PcrSelect.pcrSelections[0].sizeOfSelect =
                                                    TRUSTEDBOOT::PCR_SELECT_MAX;
    memset(o_request->PcrSelect.pcrSelections[0].pcrSelect, 0,
            sizeof(o_request->PcrSelect.pcrSelections[0].pcrSelect));

    TRUSTEDBOOT::TPM_Pcr l_pcrRegs[TRUSTEDBOOT::FW_USED_PCR_COUNT] = {
                                                             TRUSTEDBOOT::PCR_0,
                                                             TRUSTEDBOOT::PCR_1,
                                                             TRUSTEDBOOT::PCR_2,
                                                             TRUSTEDBOOT::PCR_3,
                                                             TRUSTEDBOOT::PCR_4,
                                                             TRUSTEDBOOT::PCR_5,
                                                             TRUSTEDBOOT::PCR_6,
                                                             TRUSTEDBOOT::PCR_7
                                                             };
    for(const auto l_pcr : l_pcrRegs)
    {
        o_request->PcrSelect.pcrSelections[0]
            .pcrSelect[l_pcr/TRUSTEDBOOT::FW_USED_PCR_COUNT] |=
                0x01 << (l_pcr % TRUSTEDBOOT::FW_USED_PCR_COUNT);
    }

    } while(0);

    if(l_errl)
    {
        // Error occurred. Tell the slave that the master TPM is unavailable and
        // poison the TPMs on master node.
        o_request->EyeCatcher = MSTNOTPM;
        errlHndl_t l_poisonTpmErr = TRUSTEDBOOT::poisonAllTpms();
        if(l_poisonTpmErr)
        {
            if(l_errl)
            {
                l_poisonTpmErr->plid(l_errl->plid());
            }
            if(l_tpmRequired)
            {
                errlCommit(l_poisonTpmErr, SECURE_COMP_ID);
            }
            else
            {
                TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommGenMasterQuoteRequest: "
                          "Could not poison TPMs. Errl PLID: 0x%.08X. "
                          "Deleting the error log and continuing anyway.",
                          l_poisonTpmErr->plid());
                delete l_poisonTpmErr;
                l_poisonTpmErr = nullptr;
            }
        }
    }

    TRACFCOMP(g_trac_nc, EXIT_MRK"nodeCommGenMasterQuoteRequest: " TRACE_ERR_FMT, TRACE_ERR_ARGS(l_errl));
#endif
    return l_errl;
} // nodeCommGenMasterQuoteRequest

/**
 * @brief A function to process the response of one of the slave nodes
 * @param[in] i_slaveQuote the quote received from a slave node in binary format
 * @param[in] i_slaveQuoteSize the size of the slave quote (in bytes)
 * @return nullptr on success; non-nullptr on error
 */
errlHndl_t nodeCommProcessSlaveQuote(uint8_t* const i_slaveQuote,
                                     const size_t i_slaveQuoteSize)
{
    errlHndl_t l_errl = nullptr;
#ifdef CONFIG_TPMDD
    TRACFCOMP(g_trac_nc, ENTER_MRK"nodeCommProcessSlaveQuote");
    bool l_tpmRequired = TRUSTEDBOOT::isTpmRequired();
    bool l_errorOccurred = false;

    do {
    NCEyeCatcher_t* l_eyeCatcher =
                                reinterpret_cast<NCEyeCatcher_t*>(i_slaveQuote);
    uint32_t* l_slaveNodeId = reinterpret_cast<uint32_t*>(i_slaveQuote +
                                                          sizeof(l_eyeCatcher));
    if(*l_eyeCatcher == NDNOTPM_)
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommProcessSlaveQuote: Slave node %d indicated that it could not complete the slave quote generation process", *l_slaveNodeId);
        l_errorOccurred = true;
        if(l_tpmRequired)
        {
            // Slave sent bad data and TPM is required - return an error and
            // the IPL should be terminated

            /* @
             * @errortype
             * @reasoncode RC_NC_BAD_SLAVE_QUOTE
             * @moduleid   MOD_NC_PROCESS_SLAVE_QUOTE
             * @userdata1  Slave node where the quote came from
             * @devdesc    One of the slave nodes indicated that there was
             *             an issue with its TPM or quote generation process
             * @custdesc   trustedboot failure
             */
            l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                             MOD_NC_PROCESS_SLAVE_QUOTE,
                                             RC_NC_BAD_SLAVE_QUOTE,
                                             *l_slaveNodeId,
                                             0);
            l_errl->collectTrace(SECURE_COMP_NAME);
            l_errl->collectTrace(TRBOOT_COMP_NAME);
            l_errl->collectTrace(NODECOMM_TRACE_NAME);
        }
        break;
    }

    // Extend the hash of the slave quote to PCR 1, and include the whole quote
    // in binary form as the message in the TPM log
    SHA512_t l_quoteHash = {0};
    hashBlob(i_slaveQuote, i_slaveQuoteSize, l_quoteHash);
    l_errl = TRUSTEDBOOT::pcrExtend(TRUSTEDBOOT::PCR_1,
                                    TRUSTEDBOOT::EV_PLATFORM_CONFIG_FLAGS,
                                    l_quoteHash,
                                    sizeof(l_quoteHash),
                                    i_slaveQuote,
                                    i_slaveQuoteSize);
    if(l_errl)
    {
        TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommProcessSlaveQuote: could not extend slave response to PCR 1");
        break;
    }

    } while(0);

    if(l_errl || l_errorOccurred)
    {
        errlHndl_t l_poisonTpmErr = TRUSTEDBOOT::poisonAllTpms();
        if(l_poisonTpmErr)
        {
            if(l_errl)
            {
                l_poisonTpmErr->plid(l_errl->plid());
            }
            if(TRUSTEDBOOT::isTpmRequired())
            {
                errlCommit(l_poisonTpmErr, SECURE_COMP_ID);
            }
            else
            {
                TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommProcessSlaveQuote: "
                          "Could not poison TPMs. Errl PLID: 0x%.08X. "
                          "Deleting the error log and continuing.",
                          l_poisonTpmErr->plid());
                delete l_poisonTpmErr;
                l_poisonTpmErr = nullptr;
            }
        }
    }

    TRACFCOMP(g_trac_nc, EXIT_MRK"nodeCommProcessSlaveQuote: " TRACE_ERR_FMT, TRACE_ERR_ARGS(l_errl));
#endif
    return l_errl;
} // nodeCommProcessSlaveQuote

/**
 *  @brief This function runs the procedure for the master processor on the
 *         master node to send and receive messages over the ABUS to the
 *         master processors on the slave nodes
 *
 *  @param[in] i_mProcInfo - Information about Master Proc
 *  @param[in] i_obus_instances - Vector containing all of the OBUS connections
 *                                that the Master Proc on the Master Node needs
 *                                to send and received messages across
 *
 *  @return errlHndl_t Error log handle
 *  @retval nullptr Operation was successful
 *  @retval !nullptr Operation failed with valid error log
 */
errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
                                      const std::vector<obus_instances_t> &
                                        i_obus_instances)
{
    errlHndl_t err = nullptr;

    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusExchangeMaster: mProc=0x%.08X "
              "to communicate through %d obus connection(s)",
              get_huid(i_mProcInfo.tgt), i_obus_instances.size());

    // Pointer to buffer used for receiving data from nodeCommTransferRecv()
    uint8_t * data_rcv_buffer = nullptr;

    do
    {

    // The slave quote that is returned by the slave nodes as part of the node
    // comm protocol is too big to be placed in the TPM's log without moving the
    // log to a new memory location, so before processing the slave quote, we
    // need to send a message to the TPM queue to expand the log. We need to
    // expand logs on all TPMs, since logging and PCR extends are mirrorred into
    // all TPMs/logs on the node.
    TARGETING::TargetHandleList l_tpms;
    TRUSTEDBOOT::getTPMs(l_tpms);
    for(const auto& l_tpm : l_tpms)
    {
        err = TRUSTEDBOOT::expandTpmLog(l_tpm);
        if(err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: could not expand the TPM log for TPM HUID 0x%x", TARGETING::get_huid(l_tpm));
            break;
        }
    }

    if(err)
    {
        break;
    }

    // Loop 1: Exchange SBID/nonces between Master and each of the Slave Nodes
    for ( auto const l_obus : i_obus_instances)
    {
        uint8_t my_linkId = 0;
        uint8_t my_mboxId = 0;
        getLinkMboxFromObusInstance(l_obus.myObusInstance,
                                    l_obus.myObusRelLink,
                                    my_linkId,
                                    my_mboxId);

        uint8_t expected_peer_linkId = 0;
        uint8_t expected_peer_mboxId = 0;
        getLinkMboxFromObusInstance(l_obus.peerObusInstance,
                                    // same relative link for peer path:
                                    l_obus.myObusRelLink,
                                    expected_peer_linkId,
                                    expected_peer_mboxId);

        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                  "my: linkId=%d, mboxId=%d, ObusInstance=%d. "
                  "expected peer: n%d linkId=%d, mboxId=%d, ObusInstance=%d",
                  my_linkId, my_mboxId, l_obus.myObusInstance,
                  l_obus.peerNodeInstance, expected_peer_linkId,
                  expected_peer_mboxId, l_obus.peerObusInstance);

        // Get random number from TPM
        msg_format_t msg_data;
        msg_data.value = 0;

        err = nodeCommAbusGetRandom(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                      "Error Back From nodeCommAbusGetRandom: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Set the send and expected receive LinkIds in the nonce
        msg_data.origin_linkId = my_linkId;
        msg_data.receiver_linkId = expected_peer_linkId;

        // Send a message to a slave
        size_t tmp_size = sizeof(msg_data.value);
        err = nodeCommTransferSend(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   l_obus.peerNodeInstance,
                                   NCT_TRANSFER_SBID,
                                   reinterpret_cast<uint8_t*>
                                     (&(msg_data.value)),
                                   tmp_size);

        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                      "nodeCommTransferSend returned an error");
            break;
        }

        // Push this msg_data to TPM
        err = nodeCommAbusLogNonce(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                      "Error Back From nodeCommAbusLogNonce: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Look for Return Message From The Slave
        size_t data_rcv_size = 0;
        if (data_rcv_buffer != nullptr)
        {
            free(data_rcv_buffer);
            data_rcv_buffer = nullptr;
        }
        err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   l_obus.peerNodeInstance,
                                   NCT_TRANSFER_SBID,
                                   data_rcv_buffer,
                                   data_rcv_size);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                      "nodeCommTransferRecv returned an error");
           break;
        }
        // If no err is returned, data_rcv_buffer should be valid, but do a
        // sanity check here to be certain
        assert(data_rcv_buffer!=nullptr,"nodeCommAbusExchangeMaster: Loop 1: data_rcv_buffer returned as nullptr");

        // Add receiver Link Id to the message data
        // here and in other places where SBID is handled
        memcpy(&(msg_data.value), data_rcv_buffer, data_rcv_size);
        msg_data.receiver_linkId = my_linkId;
        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Msg received, "
                  "after adding recv link (%d), 0x%.16llX will be "
                  "stored in the TPM",
                  my_linkId, msg_data.value);

        // Push this msg_data to TPM
        err = nodeCommAbusLogNonce(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 1: "
                      "Error Back From nodeCommAbusLogNonce: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

    } // end of Loop 1 (SBID/nonces) on i_obus_instances

    if(err)
    {
        break;
    }

    // Loop 2: Master Node will request quotes from each Slave Node by first
    // sending a Quote Request and then waiting to receive a Quote Response
    for ( auto const l_obus : i_obus_instances)
    {
        uint8_t my_linkId = 0;
        uint8_t my_mboxId = 0;
        getLinkMboxFromObusInstance(l_obus.myObusInstance,
                                    l_obus.myObusRelLink,
                                    my_linkId,
                                    my_mboxId);

        uint8_t expected_peer_linkId = 0;
        uint8_t expected_peer_mboxId = 0;
        getLinkMboxFromObusInstance(l_obus.peerObusInstance,
                                    // same relative link for peer path:
                                    l_obus.myObusRelLink,
                                    expected_peer_linkId,
                                    expected_peer_mboxId);

        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Loop 2: "
                  "my: linkId=%d, mboxId=%d, ObusInstance=%d. "
                  "expected peer: linkId=%d, mboxId=%d, ObusInstance=%d.",
                  my_linkId, my_mboxId, l_obus.myObusInstance,
                  expected_peer_linkId, expected_peer_mboxId,
                  l_obus.peerObusInstance);

        // Generate Master Quote Request
        MasterQuoteRequestBlob quote_request{};
        err = nodeCommGenMasterQuoteRequest(&quote_request);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 2: "
                      "Error Back From nodeCommGenMasterQuoteRequest: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Send Quote Request to a slave
        size_t tmp_size = sizeof(quote_request);
        err = nodeCommTransferSend(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   l_obus.peerNodeInstance,
                                   NCT_TRANSFER_QUOTE_REQUEST,
                                   reinterpret_cast<uint8_t*>
                                     (&quote_request),
                                   tmp_size);

        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 2: "
                      "nodeCommTransferSend returned an error");
            break;
        }

        // Look for Quote Response from the slave node
        size_t data_rcv_size = 0;
        if (data_rcv_buffer != nullptr)
        {
            free(data_rcv_buffer);
            data_rcv_buffer = nullptr;
        }
        err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   l_obus.peerNodeInstance,
                                   NCT_TRANSFER_QUOTE_RESPONSE,
                                   data_rcv_buffer,
                                   data_rcv_size);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: Loop 2: "
                      "nodeCommTransferRecv returned an error");
           break;
        }
        // If no err is returned, data_rcv_buffer should be valid, but do a
        // sanity check here to be certain
        assert(data_rcv_buffer!=nullptr,"nodeCommAbusExchangeMaster: Loop 2: data_rcv_buffer returned as nullptr");

        // Process the Quote Response that was returned from the slave node
        err = nodeCommProcessSlaveQuote(data_rcv_buffer, data_rcv_size);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeMaster: Loop 2: "
                      "Error Back nodeCommProcessSlaveQuote: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

    } // end of Loop-2 (Quotes) on i_obus_instances

    if (err)
    {
        break;
    }

    } while( 0 );

    if (data_rcv_buffer != nullptr)
    {
        free(data_rcv_buffer);
        data_rcv_buffer = nullptr;
    }

    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchangeMaster: "
              TRACE_ERR_FMT,
              TRACE_ERR_ARGS(err));

    return err;

} // end of nodeCommAbusExchangeMaster


/**
 *  @brief This function runs the procedure for the master processor on the
 *         slave nodes to receive and send messages over the ABUS to the
 *         master processor on the master node
 *
 *  @param[in] i_mProcInfo - Information about Master Proc
 *  @param[in] i_obus_instance - OBUS connection that should pertain to the
 *                               Master Proc on the master node that will send
 *                               a message to Master Proc of this slave node
 *
 *  @return errlHndl_t Error log handle
 *  @retval nullptr Operation was successful
 *  @retval !nullptr Operation failed with valid error log
 */
errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
                                     const obus_instances_t & i_obus_instance)
{
    errlHndl_t err = nullptr;

    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusExchangeSlave: mProc=0x%.08X "
              "Looking for message from master node via obus connection "
              "n%d/p%d/obus%d",
              get_huid(i_mProcInfo.tgt),
              i_obus_instance.peerNodeInstance,
              i_obus_instance.peerProcInstance,
              i_obus_instance.peerObusInstance);

    // Pointer to buffer used for transferring and receiving data
    // from nodeCommTransferSend() and nodeCommTransferRecv()
    uint8_t * data_buffer = nullptr;

    do
    {
        // Used for check that right node indicated itself as master
        uint8_t my_linkId = 0;
        uint8_t my_mboxId = 0;
        getLinkMboxFromObusInstance(i_obus_instance.myObusInstance,
                                    i_obus_instance.myObusRelLink,
                                    my_linkId,
                                    my_mboxId);


        // First Wait for Message From Master
        if (data_buffer != nullptr)
        {
            free(data_buffer);
            data_buffer = nullptr;
        }
        size_t data_size = 0;
        err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   i_obus_instance.peerNodeInstance,
                                   NCT_TRANSFER_SBID,
                                   data_buffer,
                                   data_size);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: "
                      "nodeCommTransferRecv returned an error");
           break;
        }
        // If no err is returned, data_buffer should be valid, but do a
        // sanity check here to be certain
        assert(data_buffer!=nullptr,"nodeCommAbusExchangeSlave: data_buffer returned as nullptr");


        // Add receiver Link Id to the message data
        msg_format_t msg_data;
        memcpy(&(msg_data.value), data_buffer, data_size);
        msg_data.receiver_linkId = my_linkId;

        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: Msg received, "
                  "after adding recv Link Id (%d), 0x%.16llX will "
                  "be stored in the TPM",
                  my_linkId, msg_data.value);

        // Push this msg_data to TPM
        err = nodeCommAbusLogNonce(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back "
                      "From nodeCommAbusLogNonce: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Send a message back to the master node
        // Pass in expected peer linkId for nonce logging/extending purposes
        uint8_t peer_linkId = 0;
        uint8_t peer_mboxId = 0;
        getLinkMboxFromObusInstance(i_obus_instance.peerObusInstance,
                                    // same relative link for peer path:
                                    i_obus_instance.myObusRelLink,
                                    peer_linkId,
                                    peer_mboxId);

        // Get random number from TPM
        msg_data.value = 0;
        err = nodeCommAbusGetRandom(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back "
                      "From nodeCommAbusGetRandom: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Set the send and expected receive LinkIds in the nonce
        msg_data.origin_linkId = my_linkId;
        msg_data.receiver_linkId = my_linkId;

        // Send a message to a slave
        size_t tmp_size = sizeof(msg_data.value);
        err = nodeCommTransferSend(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   i_obus_instance.peerNodeInstance,
                                   NCT_TRANSFER_SBID,
                                   reinterpret_cast<uint8_t*>
                                     (&(msg_data.value)),
                                   tmp_size);

        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
                      "nodeCommAbusTransferSend returned an error");
            break;
        }

        // Push this msg_data to TPM
        err = nodeCommAbusLogNonce(msg_data.value);
        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: Error Back From "
                      "nodeCommAbusLogNonce: "
                      TRACE_ERR_FMT,
                      TRACE_ERR_ARGS(err));
            break;
        }

        // Wait for Quote Request from Master Node
        if (data_buffer != nullptr)
        {
            free(data_buffer);
            data_buffer = nullptr;
        }
        data_size = 0;
        err = nodeCommTransferRecv(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   i_obus_instance.peerNodeInstance,
                                   NCT_TRANSFER_QUOTE_REQUEST,
                                   data_buffer,
                                   data_size);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: "
                      "nodeCommTransferRecv returned an error");
           break;
        }
        // If no err is returned, data_buffer should be valid, but do a
        // sanity check here to be certain
        assert(data_buffer!=nullptr,"nodeCommAbusExchangeSlave: data_buffer returned as nullptr");

        // Cast the data received into a MasterQuoteRequestBlob
        MasterQuoteRequestBlob quote_request{};
        memcpy(&quote_request, data_buffer, data_size);

        // re-use data_buffer and data_size for the Quote Response
        if (data_buffer != nullptr)
        {
            free(data_buffer);
            data_buffer = nullptr;
        }
        data_size = 0;

        // Generate the Quote Response
        err = nodeCommGenSlaveQuoteResponse(&quote_request,
                                            data_size,
                                            data_buffer);

        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeSlave: "
                      "nodeCommGenSlaveQuoteResponse returned an error");
           break;
        }

        // Send the Quote Response
        err = nodeCommTransferSend(i_mProcInfo.tgt,
                                   my_linkId,
                                   my_mboxId,
                                   i_obus_instance.peerNodeInstance,
                                   NCT_TRANSFER_QUOTE_RESPONSE,
                                   data_buffer,
                                   data_size);

        if (err)
        {
            TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchangeSlave: "
                      "nodeCommTransferSend returned an error");
            break;
        }

        // Nothing left for slave node to do

    } while( 0 );

    if (data_buffer != nullptr)
    {
        free(data_buffer);
        data_buffer = nullptr;
    }


    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchangeSlave: "
              TRACE_ERR_FMT,
              TRACE_ERR_ARGS(err));

    return err;

} // end of nodeCommAbusExchangeSlave


/**
 *  @brief Runs the procedure for the drawers/nodes to exchange messages
 *         over the ABUS Link Mailbox facility
 *
 *  Defined in nodecommif.H
 */
errlHndl_t nodeCommAbusExchange(void)
{
    errlHndl_t err = nullptr;

#ifndef CONFIG_TPMDD
    TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: Not running procedure "
              "since CONFIG_TPMDD is not set");
#else


    master_proc_info_t mProcInfo;
    std::vector<obus_instances_t> obus_instances;
    char * l_phys_path_str = nullptr;
    char * l_peer_path_str = nullptr;

    TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusExchange:");

    do
    {
    // Get Target Service, and the system target.
    TargetService& tS = targetService();
    Target* sys = nullptr;
    (void) tS.getTopLevelTarget( sys );
    assert(sys, "nodeCommAbusExchange: system target is NULL");


    // Get some info about the nodes in the system
    auto hb_images = sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>();
    const int total_nodes = __builtin_popcount(hb_images);
    uint64_t my_nodeid = TARGETING::UTIL::getCurrentNodePhysId();
    size_t my_round = 0;

    // Create mask to use for check later
    ATTR_HB_EXISTING_IMAGE_type mask = 0x1 <<
        ((sizeof(ATTR_HB_EXISTING_IMAGE_type) * 8) -1);

    for (size_t l_node=0, count=0;
         (l_node < MAX_NODES_PER_SYS);
         ++l_node )
    {
        if( 0 != ((mask >> l_node) & hb_images ) )
        {
            ++count;
            if (l_node == my_nodeid)
            {
                my_round = count;
                TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                          "this node is position %d of %d total nodes "
                          "(l_nodeId=%d, hb_existing_image=0x%X)",
                          my_round, total_nodes, my_nodeid, hb_images );
                break;
            }
        }
    }

    // Get master proc for this node
    err = tS.queryMasterProcChipTargetHandle(mProcInfo.tgt);
    if (err)
    {
        TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommAbusExchange: "
                  "queryMasterProcChipTargetHandle returned error: "
                  TRACE_ERR_FMT,
                  TRACE_ERR_ARGS(err));
        break;
    }

    // Extract info from the master proc of this node
    EntityPath l_PHYS_PATH = mProcInfo.tgt->getAttr<ATTR_PHYS_PATH>();
    EntityPath::PathElement l_peMasterNode =
                              l_PHYS_PATH.pathElementOfType(TYPE_NODE);
    EntityPath::PathElement l_peMasterProc =
                              l_PHYS_PATH.pathElementOfType(TYPE_PROC);
    l_phys_path_str = l_PHYS_PATH.toString();

    if((l_peMasterNode.type == TYPE_NA) ||
       (l_peMasterProc.type == TYPE_NA))
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchange: "
                  "ERR: Cannot find NODE or PROC in masterProc 0x%.08X "
                  "PHYS_PATH %s",
                  get_huid(mProcInfo.tgt),
                  l_phys_path_str);
        /*@
         * @errortype
         * @reasoncode       RC_NCEX_INVALID_PHYS_PATH
         * @moduleid         MOD_NCEX_MAIN
         * @userdata1        Master Proc Target HUID
         * @userdata2        <Unused>
         * @devdesc          Master Proc's ATTR_PHYS_PATH is invalid as it
         *                   doesn't have either a NODE or PROC elemenent
         * @custdesc         Trusted Boot failure
         */
        err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                       MOD_NCEX_MAIN,
                                       RC_NCEX_INVALID_PHYS_PATH,
                                       get_huid(mProcInfo.tgt),
                                       0,
                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );

        ERRORLOG::ErrlUserDetailsStringSet path;
        path.add("mProc PHYS Entity Path", l_phys_path_str);
        path.addToLog(err);

        break;
    }
    mProcInfo.nodeInstance = l_peMasterNode.instance;
    mProcInfo.procInstance = l_peMasterProc.instance;
    TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
              "Master Proc 0x%.08X: PHYS_PATH=%s: "
              "nodeInstance=%d, procInstance=%d",
              get_huid(mProcInfo.tgt),
              l_phys_path_str,
              mProcInfo.nodeInstance,
              mProcInfo.procInstance);

    // Walk Through OBUS Chiplets on the Master Proc
    TargetHandleList l_obusTargetList;
    getChildChiplets(l_obusTargetList, mProcInfo.tgt, TYPE_OBUS);

    TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: proc 0x%.08X has "
              "%d functional OBUS Chiplets",
              get_huid(mProcInfo.tgt), l_obusTargetList.size());

    // Loop through OBUS Targets and evaluate their PEER_TARGETs
    for (const auto & l_obusTgt : l_obusTargetList)
    {

        EntityPath l_peerPath = l_obusTgt->getAttr<ATTR_PEER_PATH>();
        EntityPath::PathElement l_peProc =
                                  l_peerPath.pathElementOfType(TYPE_PROC);

        if (l_peer_path_str != nullptr)
        {
            free(l_peer_path_str);
        }
        l_peer_path_str = l_peerPath.toString();


        if(l_peProc.type == TYPE_NA)
        {
            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X "
                      "OBUS HUID 0x%.08X's PEER_PATH %s because "
                      "cannot find PROC in PEER_PATH",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str);
            continue;
        }

        // check that proc has same position as our master
        if (l_peProc.instance != mProcInfo.procInstance)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X "
                      "OBUS HUID 0x%.08X's PEER_PATH %s because PROC "
                      "Instance=%d does not match masterProc Instance=%d",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str, l_peProc.instance,
                      mProcInfo.procInstance);
            continue;
        }

        EntityPath::PathElement l_peNode =
                                  l_peerPath.pathElementOfType(TYPE_NODE);
        if(l_peNode.type == TYPE_NA)
        {
            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X "
                      "OBUS HUID 0x%.08X's PEER_PATH %s because "
                      "cannot find NODE in PEER_PATH",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str);
            continue;
        }

        // Check that node exists and isn't this node
        if (!((mask >> l_peNode.instance) & hb_images) ||
             (l_peNode.instance == my_nodeid))
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X "
                      "OBUS HUID 0x%.08X's PEER_PATH %s because either "
                      "Node=%d is not configured or is this node (%d)",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str, l_peNode.instance, my_nodeid);
            continue;
        }

        obus_instances_t l_obusInstance;

        EntityPath::PathElement l_peObus =
                                  l_peerPath.pathElementOfType(TYPE_OBUS);
        if(l_peObus.type == TYPE_NA)
        {
            TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X "
                      "OBUS HUID 0x%.08X's PEER_PATH %s because "
                      "cannot find OBUS in PEER_PATH",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str);
            continue;
        }

        bool link0_trained = false;
        bool link1_trained = false;
        uint64_t fir_data = 0;
        err = getObusTrainedLinks(l_obusTgt,
                                  link0_trained,
                                  link1_trained,
                                  fir_data);
        if (err)
        {
            TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommAbusExchange: "
                      "getObusTrainedLinks returned error so "
                      "Skipping masterProc 0x%.08X OBUS HUID 0x%.08X. "
                      TRACE_ERR_FMT,
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      TRACE_ERR_ARGS(err));
            break;
        }
        else
        {
            TRACFCOMP(g_trac_nc, INFO_MRK"nodeCommAbusExchange: "
                      "getObusTrainedLinks: link0=%d, link1=%d",
                      link0_trained, link1_trained);

            l_obusInstance.myObusRelLink = (link0_trained) ? 0 :
                                             ((link1_trained) ? 1 :
                                               NCDD_INVALID_LINK_MBOX);

            if (l_obusInstance.myObusRelLink == NCDD_INVALID_LINK_MBOX)
            {
               TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                         "Skipping masterProc 0x%.08X "
                         "OBUS HUID 0x%.08X's because neither link has been "
                         "trained: link0=%d, link1=%d (fir_data=0x%.16llX)",
                         get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                         link0_trained, link1_trained, fir_data);
                continue;
            }

        }

        // Using this OBUS instance so save it off
        l_obusInstance.peerObusInstance = l_peObus.instance;
        l_obusInstance.peerProcInstance = l_peProc.instance;
        l_obusInstance.peerNodeInstance = l_peNode.instance;
        l_obusInstance.myObusInstance = l_obusTgt->getAttr<ATTR_REL_POS>();

        // Before adding to list check that on a 2-node system that we ignore
        // redundant connections to the same node
        if (total_nodes==2)
        {
            bool l_duplicate_found = false;
            for (auto l_saved_instance : obus_instances)
            {
                if (l_saved_instance.peerNodeInstance ==
                    l_obusInstance.peerNodeInstance)
                {
                    TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "Skipping masterProc 0x%.08X OBUS HUID 0x%.08X's "
                      "PEER_PATH %s because already have saved connection to "
                      "that node: myObusInstance=%d, peerInstance:n%d/p%d/obus%d",
                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                      l_peer_path_str,
                      l_saved_instance.myObusInstance,
                      l_saved_instance.peerNodeInstance,
                      l_saved_instance.peerProcInstance,
                      l_saved_instance.peerObusInstance);

                    l_duplicate_found = true;
                    break;
                }
            }
            if (l_duplicate_found == true)
            {
                 // continue to skip adding this instance to obus_instances
                 continue;
            }
        }

        obus_instances.push_back(l_obusInstance);
        TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: Using masterProc 0x%.08X "
                  "OBUS HUID 0x%.08X's peer path %s with obus_instance "
                  "myObusInstance=%d, peer=n%d/p%d/obus%d (vector size=%d)",
                  get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
                  l_peer_path_str, l_obusInstance.myObusInstance,
                  l_obusInstance.peerNodeInstance,
                  l_obusInstance.peerProcInstance,
                  l_obusInstance.peerObusInstance, obus_instances.size());
    }

    // If invalid number of peer paths fail
    if((obus_instances.size() == 0) ||
       (obus_instances.size() != (static_cast<uint32_t>(total_nodes)-1)))
    {
        TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusExchange: "
                  "ERR: Invalid number of PEER_PATHs %d found for proc=0x%.08X "
                  "when there are %d nodes in the system",
                  obus_instances.size(), get_huid(mProcInfo.tgt),
                  total_nodes);

        /*@
         * @errortype
         * @reasoncode       RC_NCEX_INVALID_INSTANCE_COUNT
         * @moduleid         MOD_NCEX_MAIN
         * @userdata1        Master Proc Target HUID
         * @userdata2[0:31]  Number of Valid OBUS Instances
         * @userdata2[32:63] Total Number Of Nodes
         * @devdesc          When processing the OBUS Peer paths, the wrong
         *                   count of valid paths was found
         * @custdesc         Trusted Boot failure
         */
        err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                       MOD_NCEX_MAIN,
                                       RC_NCEX_INVALID_INSTANCE_COUNT,
                                       get_huid(mProcInfo.tgt),
                                       TWO_UINT32_TO_UINT64(
                                         obus_instances.size(),
                                         total_nodes),
                                       ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );

        break;
    }


    // Master node should have lowest node number in vector of instance info.
    // So sort here by node number and then only pass first instance to
    // nodeCommAbusExchangeSlave() below.  Then when message is received it
    // will be checked against the expected master instance.
    std::sort(obus_instances.begin(),
              obus_instances.end(),
              [](obus_instances_t & lhs,
                 obus_instances_t & rhs)
    {
        return lhs.peerNodeInstance < rhs.peerNodeInstance;
    });


    // @TODO RTC 194053 - This won't work on simics until the action files
    // are there
    if (Util::isSimicsRunning())
    {
        TRACFCOMP(g_trac_nc,"nodeCommAbusExchange: actual operation is not "
                  "supported on simics yet");
        break;
    }

    if (TARGETING::UTIL::isCurrentMasterNode())
    {
        err = nodeCommAbusExchangeMaster(mProcInfo,obus_instances);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "nodeCommAbusExchangeMaster returned an error");
           break;
        }
    }
    else
    {
        // Only pass first instance, which should be master instance
        err = nodeCommAbusExchangeSlave(mProcInfo,
                                        obus_instances[0]);
        if (err)
        {
           TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
                      "nodeCommAbusExchangeSlave returned an error");
           break;
        }
    }

    if(err)
    {
        break;
    }

    } while( 0 );

    TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchange: %s: "
              TRACE_ERR_FMT,
              (err == nullptr) ? "SUCCESSFUL" : "FAILED",
              TRACE_ERR_ARGS(err));

    if (err)
    {
        if(!TRUSTEDBOOT::isTpmRequired())
        {
            TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusExchange:An error occurred"
                     " during secure node communication, but the TPM required "
                     "policy is not set, so the error will not be propagated."
                     " Original error RC: 0x%.04X; PLID: 0x%.08X."
                     " Deleting the error log and continuing.",
                     err->reasonCode(),
                     err->plid());
            delete err;
            err = nullptr;
        }
        else
        {
            err->collectTrace(SECURE_COMP_NAME);
            err->collectTrace(NODECOMM_TRACE_NAME);
            err->collectTrace(TRBOOT_COMP_NAME);
        }
    }

    if (l_phys_path_str != nullptr)
    {
        free(l_phys_path_str);
        l_phys_path_str = nullptr;
    }
    if (l_peer_path_str != nullptr)
    {
        free(l_peer_path_str);
        l_peer_path_str = nullptr;
    }

#endif

    return err;

} // end of nodeCommAbusExchange

} // End NODECOMM namespace

} // End SECUREBOOT namespace