ABUS Node Communication: Add Support for Redundant Links

This commit adds redundancy to the ABUS Node Communication procedure
by using the best possible link per connection rather than the default
secure link.

Change-Id: I051d316eacb05d59889d3f45fcb387e29877e1fe
RTC:184518
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/62133
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

3 files changed, 257 insertions, 43 deletions

diff --git a/src/usr/secureboot/node_comm/node_comm.C b/src/usr/secureboot/node_comm/node_comm.C
index 95008344a..cb6b18bc9 100644
--- a/src/usr/secureboot/node_comm/node_comm.C
+++ b/src/usr/secureboot/node_comm/node_comm.C
@@ -413,6 +413,81 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
 
 } // end of nodeCommMapAttn
 
+/**
+ *  @brief For OBUS OLL FIR Register (0x9010800) bit0 represents whether or
+ *         not link0 has been trained and bit1 represents whether or not
+ *         link1 has been trained
+ */
+enum link_trained_values : uint64_t
+{
+    // Register to read to find out which links on an OBUS are trained
+    OLL_FIR_REGISTER      = 0x0000000009010800,
+
+    // Bit Mask values
+    IS_LINK0_TRAINED_MASK = 0x8000000000000000,
+    IS_LINK1_TRAINED_MASK = 0x4000000000000000,
+};
+
+/**
+ *  @brief Return the status of whether or not the 2 links connected to the
+ *         OBUS Chiplet are trained
+ */
+errlHndl_t getObusTrainedLinks(TARGETING::Target* i_pObus,
+                               bool & o_link0_trained,
+                               bool & o_link1_trained,
+                               uint64_t & o_fir_data)
+{
+    errlHndl_t err = nullptr;
+    o_link0_trained = false;
+    o_link1_trained = false;
+    o_fir_data = 0;
+    const uint64_t fir_addr = OLL_FIR_REGISTER;
+    const size_t expSize = sizeof(o_fir_data);
+
+    assert(i_pObus != nullptr, "getObusTrainedLinks: i_pObus == nullptr");
+
+    TRACUCOMP(g_trac_nc,ENTER_MRK
+              "getObusTrainedLinks: OBUS tgt=0x%X",
+              get_huid(i_pObus));
+
+    do
+    {
+    // Read the OBUS OLL FIR Register
+    auto reqSize = expSize;
+    err = DeviceFW::deviceRead(i_pObus,
+                               &o_fir_data,
+                               reqSize,
+                               DEVICE_SCOM_ADDRESS(fir_addr));
+
+    if(err)
+    {
+        TRACFCOMP(g_trac_nc,ERR_MRK"getObusTrainedLinks: Read Fail! "
+                  "tgt=0x%X, fir_addr=0x%.16llX, fir_data=0x%.16llX "
+                  TRACE_ERR_FMT,
+                  TARGETING::get_huid(i_pObus),
+                  fir_addr, o_fir_data,
+                  TRACE_ERR_ARGS(err));
+        break;
+    }
+    assert(reqSize==expSize,"getObusTrainedLinks: SCOM deviceRead didn't return expected data size of %d (it was %d)",
+           expSize,reqSize);
+
+    o_link0_trained = (o_fir_data & IS_LINK0_TRAINED_MASK) != 0;
+    o_link1_trained = (o_fir_data & IS_LINK1_TRAINED_MASK) != 0;
+
+
+    } while( 0 );
+
+    TRACFCOMP(g_trac_nc,EXIT_MRK"getObusTrainedLinks: OBUS tgt=0x%X: "
+              "o_link0_trained=%d, o_link1_trained=%d (fir_data=0x%.16llX) "
+              TRACE_ERR_FMT,
+              get_huid(i_pObus), o_link0_trained, o_link1_trained, o_fir_data,
+              TRACE_ERR_ARGS(err));
+
+    return err;
+
+} // end of getObusTrainedLinks
+
 
 /**
  * @brief Add FFDC for the target to an error log
@@ -526,7 +601,7 @@ void addNodeCommBusCallout(node_comm_modes_t i_mode,
     TargetHandleList l_busTargetList;
     TYPE l_type = (i_mode == NCDD_MODE_ABUS)
                    ? TYPE_OBUS : TYPE_XBUS;
-    getChildChiplets(l_busTargetList, i_pProc, l_type);
+    getChildChiplets(l_busTargetList, i_pProc, l_type, false);
 
     // Get BUS Instance
     // For each OBUS or XBUS instance there are 2 links and 2 mailboxes,
@@ -575,24 +650,90 @@ void addNodeCommBusCallout(node_comm_modes_t i_mode,
 
         if (l_bus_instance_ep1 == l_busTgt->getAttr<ATTR_REL_POS>())
         {
-            TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
-                      "Using i_pProc 0x%.08X BUS HUID 0x%.08X (%s) "
-                      "PEER_PATH %s as it had right instance %d for ep1",
-                      get_huid(i_pProc), get_huid(l_busTgt), l_ep1_path_str,
-                      l_ep2_path_str, l_bus_instance_ep1);
-
-            found_peer_endpoint = true;
-
-            // Add Bus Callout
-            io_log->addBusCallout(l_ep1,
-                                  l_ep2,
-                                  l_bus_type,
-                                  i_priority);
-
-           // @TODO RTC 184518 - Possibly go to depth of TYPE_SMPGROUP,
-           // which is one level below OBUS for ABUS/OBUS callouts
-
-            break;
+            // If XBUS, make callout here:
+            if (i_mode == NCDD_MODE_XBUS)
+            {
+                TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                          "Using i_pProc 0x%.08X BUS HUID 0x%.08X (%s) "
+                          "PEER_PATH %s as it had right instance %d for ep1",
+                          get_huid(i_pProc), get_huid(l_busTgt), l_ep1_path_str,
+                          l_ep2_path_str, l_bus_instance_ep1);
+
+                found_peer_endpoint = true;
+
+                // Add Bus Callout
+                io_log->addBusCallout(l_ep1,
+                                      l_ep2,
+                                      l_bus_type,
+                                      i_priority);
+               break;
+            }
+            else
+            {
+                // Go to depth of TYPE_SMPGROUP, which is one level below OBUS
+                TargetHandleList l_smpGroupTargetList;
+                getChildAffinityTargets(l_smpGroupTargetList,
+                                        l_busTgt,
+                                        CLASS_UNIT,
+                                        TYPE_SMPGROUP,
+                                        false);
+
+                for (auto l_smpGroup : l_smpGroupTargetList)
+                {
+                    EntityPath l_smpGroup_ep =
+                                 l_smpGroup->getAttr<ATTR_PHYS_PATH>();
+                    EntityPath::PathElement l_smpGroup_ep_peSmpGroup =
+                                 l_smpGroup_ep.pathElementOfType(TYPE_SMPGROUP);
+                    if (l_ep1_path_str != nullptr)
+                    {
+                         free(l_ep1_path_str);
+                    }
+                    l_ep1_path_str = l_smpGroup_ep.toString();
+
+                    EntityPath l_smpGroup_peer_ep =
+                                 l_smpGroup->getAttr<ATTR_PEER_PATH>();
+                    if (l_ep2_path_str != nullptr)
+                    {
+                        free(l_ep2_path_str);
+                    }
+                    l_ep2_path_str = l_smpGroup_peer_ep.toString();
+
+                    TRACUCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                              "SMPGROUP HUID: 0x%.08X (%s): rel_pos=%d, "
+                              "instance=%d: peer=%s",
+                              get_huid(l_smpGroup),
+                              l_ep1_path_str,
+                              l_smpGroup->getAttr<ATTR_REL_POS>(),
+                              l_smpGroup_ep_peSmpGroup.instance,
+                              l_ep2_path_str);
+
+                    // Find matching instance and relative link Id
+                    if ((l_smpGroup_ep_peSmpGroup.instance % 2) == 
+                        (i_linkId % 2))
+                    {
+                        found_peer_endpoint = true;
+
+                        TRACFCOMP(g_trac_nc,INFO_MRK"addNodeCommBusCallout: "
+                                  "Using SMPGROUP HUID: 0x%.08X (%s): "
+                                  "i_linkId=%d, rel_pos=%d, instance=%d: "
+                                  "peer=%s",
+                                  get_huid(l_smpGroup),
+                                  l_ep1_path_str,
+                                  i_linkId,
+                                  l_smpGroup->getAttr<ATTR_REL_POS>(),
+                                  l_smpGroup_ep_peSmpGroup.instance,
+                                  l_ep2_path_str);
+
+                        // Add Bus Callout
+                        io_log->addBusCallout(l_smpGroup_ep,
+                                              l_smpGroup_peer_ep,
+                                              l_bus_type,
+                                              i_priority);
+                        break;
+                    }
+                }  // for loop on SMPGROUP
+
+            }
         }
         else
         {
@@ -604,7 +745,12 @@ void addNodeCommBusCallout(node_comm_modes_t i_mode,
                       l_ep2_path_str, l_busTgt->getAttr<ATTR_ORDINAL_ID>(),
                       l_bus_instance_ep1, i_linkId);
         }
-    }
+
+        if (found_peer_endpoint == true)
+        {
+            break;
+        }
+    }  // for loop for OBUS
 
     if (found_peer_endpoint == false)
     {
diff --git a/src/usr/secureboot/node_comm/node_comm.H b/src/usr/secureboot/node_comm/node_comm.H
index 7ed48ffc9..5a6b7e684 100644
--- a/src/usr/secureboot/node_comm/node_comm.H
+++ b/src/usr/secureboot/node_comm/node_comm.H
@@ -133,20 +133,22 @@ inline uint64_t getLinkMboxReg(uint8_t i_linkId, uint8_t i_mboxId)
 /**
  *  @brief Calculate Secure LinkId and MailboxId based on OBUS Instance
  *
- *  @param[out] i_obusInstance - OBUS Instance of the operation
+ *  @param[in]  i_obusInstance - OBUS instance of the operation
+ *  @param[in]  i_obusRelLink  - Relative Link (0 or 1) of the OBUS instance
  *  @param[out] o_linkId       - Link Id of the OBUS instance
  *  @param[out] o_mboxId       - Mailbox Id of the OBUS instance
  *
  *  @return void
  */
-inline void getSecureLinkMboxFromObus(uint8_t i_obusInstance,
-                                      uint8_t & o_linkId,
-                                      uint8_t & o_mboxId)
+inline void getLinkMboxFromObusInstance(uint8_t i_obusInstance,
+                                        uint8_t i_obusRelLink,
+                                        uint8_t & o_linkId,
+                                        uint8_t & o_mboxId)
 {
     // For each OBUS instance there are 2 links and 2 mailboxes
     // and the Secure Link and Mailbox is always the relative 'L0M0'
     // of these 4 possibilities
-    o_linkId = (2*i_obusInstance);
+    o_linkId = (2*i_obusInstance) + i_obusRelLink;
     o_mboxId = 0;
 
     return;
@@ -218,12 +220,13 @@ errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
 /**
  *  @brief Map Attention Bits in XBUS/ABUS FIR Register to specific Link Mailbox
  *
- *  @param[in] i_pProc      Processor target to look for attentions on
- *                          Can't be nullptr
- *  @param[in] i_mode       Indicates to look for ABUS or XBUS attentions
- *  @param[in] o_attn_found Returns true if attention was found; otherwise false
- *  @param[in] o_linkId     Link Id attention was found on
- *  @param[in] o_mboxId     Mbox Id attention was found on
+ *  @param[in] i_pProc       Processor target to look for attentions on
+ *                           Can't be nullptr
+ *  @param[in] i_mode        Indicates to look for ABUS or XBUS attentions
+ *  @param[out] o_attn_found Returns true if attention was found;
+ *                           otherwise false
+ *  @param[out] o_linkId     Link Id attention was found on
+ *  @param[out] o_mboxId     Mbox Id attention was found on
  *  @note - o_linkId and o_mboxId are only set/valid if o_attn_found is true
  *
  *  @return errlHndl_t Error log handle
@@ -235,6 +238,24 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
                            bool & o_attn_found,
                            uint8_t & o_linkId,
                            uint8_t & o_mboxId);
+/**
+ *  @brief Return the status of whether or not the 2 links connected to the
+ *         OBUS Chiplet are trained
+ *
+ *  @param[in] i_pObus           Target representing OBUS Chiplet
+ *                               Can't be nullptr
+ *  @param[out] o_link0_trained  True if link0 is trained; otherwise false
+ *  @param[out] o_link1_trained  True is link1 is trained; otherwise false
+ *  @param[out] o_fir_data       The SCOM data read from the FIR register
+ *
+ *  @return errlHndl_t Error log handle
+ *  @retval nullptr Operation was successful
+ *  @retval !nullptr Operation failed with valid error log
+ */
+errlHndl_t getObusTrainedLinks(TARGETING::Target* i_pObus,
+                               bool & o_link0_trained,
+                               bool & o_link1_trained,
+                               uint64_t & o_fir_data);
 
 /**
  * @brief Add FFDC for the target to an error log
diff --git a/src/usr/secureboot/node_comm/node_comm_exchange.C b/src/usr/secureboot/node_comm/node_comm_exchange.C
index 49fd5a36b..882d4f9f3 100644
--- a/src/usr/secureboot/node_comm/node_comm_exchange.C
+++ b/src/usr/secureboot/node_comm/node_comm_exchange.C
@@ -86,6 +86,7 @@ struct master_proc_info_t
 struct obus_instances_t
 {
     uint8_t  myObusInstance = 0;
+    uint8_t  myObusRelLink  = 0;
 
     // Expect data to be received on the PEER_PATH instances
     uint8_t  peerNodeInstance = 0;
@@ -290,15 +291,18 @@ errlHndl_t nodeCommAbusExchangeMaster(const master_proc_info_t & i_mProcInfo,
     {
         uint8_t my_linkId = 0;
         uint8_t my_mboxId = 0;
-        getSecureLinkMboxFromObus(l_obus.myObusInstance,
-                                  my_linkId,
-                                  my_mboxId);
+        getLinkMboxFromObusInstance(l_obus.myObusInstance,
+                                    l_obus.myObusRelLink,
+                                    my_linkId,
+                                    my_mboxId);
 
         uint8_t expected_peer_linkId = 0;
         uint8_t expected_peer_mboxId = 0;
-        getSecureLinkMboxFromObus(l_obus.peerObusInstance,
-                                  expected_peer_linkId,
-                                  expected_peer_mboxId);
+        getLinkMboxFromObusInstance(l_obus.peerObusInstance,
+                                    // same relative link for peer path:
+                                    l_obus.myObusRelLink,
+                                    expected_peer_linkId,
+                                    expected_peer_mboxId);
 
         TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchangeMaster: "
                   "my: linkId=%d, mboxId=%d, ObusInstance=%d. "
@@ -494,9 +498,10 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         // Used for check that right node indicated itself as master
         uint8_t my_linkId = 0;
         uint8_t my_mboxId = 0;
-        getSecureLinkMboxFromObus(i_obus_instance.myObusInstance,
-                                  my_linkId,
-                                  my_mboxId);
+        getLinkMboxFromObusInstance(i_obus_instance.myObusInstance,
+                                    i_obus_instance.myObusRelLink,
+                                    my_linkId,
+                                    my_mboxId);
 
         // First Wait for Message From Master
         uint8_t actual_linkId = 0;
@@ -598,9 +603,11 @@ errlHndl_t nodeCommAbusExchangeSlave(const master_proc_info_t & i_mProcInfo,
         // Pass in expected peer linkId for nonce logging/extending purposes
         uint8_t peer_linkId = 0;
         uint8_t peer_mboxId = 0;
-        getSecureLinkMboxFromObus(i_obus_instance.peerObusInstance,
-                                  peer_linkId,
-                                  peer_mboxId);
+        getLinkMboxFromObusInstance(i_obus_instance.peerObusInstance,
+                                    // same relative link for peer path:
+                                    i_obus_instance.myObusRelLink,
+                                    peer_linkId,
+                                    peer_mboxId);
 
         // Get random number from TPM
         msg_data.value = 0;
@@ -859,6 +866,46 @@ errlHndl_t nodeCommAbusExchange(void)
             continue;
         }
 
+        bool link0_trained = false;
+        bool link1_trained = false;
+        uint64_t fir_data = 0;
+        err = getObusTrainedLinks(l_obusTgt,
+                                  link0_trained,
+                                  link1_trained,
+                                  fir_data);
+        if (err)
+        {
+            TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommAbusExchange: "
+                      "getObusTrainedLinks returned error so "
+                      "Skipping masterProc 0x%.08X OBUS HUID 0x%.08X. "
+                      TRACE_ERR_FMT,
+                      get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
+                      TRACE_ERR_ARGS(err));
+            break;
+        }
+        else
+        {
+            TRACFCOMP(g_trac_nc, ERR_MRK"nodeCommAbusExchange: "
+                      "getObusTrainedLinks: link0=%d, link1=%d",
+                      link0_trained, link1_trained);
+
+            l_obusInstance.myObusRelLink = (link0_trained) ? 0 :
+                                             ((link1_trained) ? 1 :
+                                               NCDD_INVALID_LINK_MBOX);
+
+            if (l_obusInstance.myObusRelLink == NCDD_INVALID_LINK_MBOX)
+            {
+               TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusExchange: "
+                         "Skipping masterProc 0x%.08X "
+                         "OBUS HUID 0x%.08X's because neither link has been "
+                         "trained: link0=%d, link1=%d (fir_data=0x%.16llX)",
+                         get_huid(mProcInfo.tgt), get_huid(l_obusTgt),
+                         link0_trained, link1_trained, fir_data);
+                continue;
+            }
+
+        }
+
         // Using this OBUS instance so save it off
         l_obusInstance.peerObusInstance = l_peObus.instance;
         l_obusInstance.peerProcInstance = l_peProc.instance;