L2 - Fabric updates for multi-chip support

  Refactor p9_build_smp code
    Add FBC effective config (attribute-only) HWPs
    Add/call FBC initfiles

  HWP flow
    p9_fbc_eff_config
    p9_fbc_eff_config_links
    p9_chiplet_scominit
    p9_smp_link_layer
    p9_fab_iovalid
    p9_fbc_eff_config_aggregate
    p9_build_smp

  Update engd used in build to e9035 u087

Change-Id: I6b6e1291ac64a9b55169eb0a077801bec41867d9
Original-Change-Id: I9ab9e967847d380596e896a14e481ad8cf247b9a
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/21643
Tested-by: PPE CI
Tested-by: Hostboot CI
Tested-by: Jenkins Server
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: Benjamin Gass <bgass@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/60646
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Disable-CI: Daniel M. Crowell <dcrowell@us.ibm.com>
Tested-by: Daniel M. Crowell <dcrowell@us.ibm.com>

1 files changed, 283 insertions, 61 deletions

diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
index f51357db2..097dfbe7a 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
@@ -41,25 +41,207 @@
 // Includes
 //------------------------------------------------------------------------------
 #include <p9_fab_iovalid.H>
-#include <p9_fbc_utils.H>
+#include <p9_fbc_smp_utils.H>
 
 
 //------------------------------------------------------------------------------
 // Constant definitions
 //------------------------------------------------------------------------------
+// EXTFIR/RAS FIR field constants
 const uint8_t IOVALID_FIELD_NUM_BITS = 2;
 
+// DL FIR register field constants
+const uint8_t DL_FIR_LINK0_TRAINED_BIT = 0;
+const uint8_t DL_FIR_LINK1_TRAINED_BIT = 1;
+
+// TL FIR register field constants
+const uint8_t TL_FIR_TRAINED_FIELD_LENGTH = 2;
+const uint8_t TL_FIR_TRAINED_LINK_TRAINED = 0x3;
+
+// TL Link Delay register field constants
+const uint8_t TL_LINK_DELAY_FIELD_NUM_BITS = 12;
+
 
 //------------------------------------------------------------------------------
 // Function definitions
 //------------------------------------------------------------------------------
 
+
+/// @brief Validate DL/TL link layers are trained
+///
+/// @param[in]  i_target          Processor chip target
+///
+/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+fapi2::ReturnCode p9_fab_iovalid_link_validate(
+    const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+    const p9_fbc_link_ctl_t& i_link_ctl)
+{
+    FAPI_DBG("Start");
+    fapi2::buffer<uint64_t> l_dl_fir_reg;
+    fapi2::buffer<uint64_t> l_tl_fir_reg;
+    uint8_t l_tl_fir_trained_state = 0;
+
+    // validate DL training state
+    FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.dl_fir_addr, l_dl_fir_reg),
+             "Error from getScom (0x%.16llX)", i_link_ctl.dl_fir_addr);
+    FAPI_ASSERT(l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>() &&
+                l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>(),
+                fapi2::P9_FAB_IOVALID_DL_NOT_TRAINED_ERR()
+                .set_TARGET(i_target)
+                .set_LOC_ENDP_TYPE(i_link_ctl.endp_type)
+                .set_LOC_ENDP_UNIT_ID(i_link_ctl.endp_unit_id)
+                .set_DL_FIR_REG(l_dl_fir_reg),
+                "Link DL training did not complete successfully!");
+
+    // validate TL training state
+    FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.tl_fir_addr, l_tl_fir_reg),
+             "Error from getScom (0x%.16llX)", i_link_ctl.tl_fir_addr);
+    FAPI_TRY(l_tl_fir_reg.extractToRight(l_tl_fir_trained_state,
+                                         i_link_ctl.tl_fir_trained_field_start_bit,
+                                         TL_FIR_TRAINED_FIELD_LENGTH),
+             "Error extracting TL layer training state");
+
+    FAPI_ASSERT(l_tl_fir_trained_state == TL_FIR_TRAINED_LINK_TRAINED,
+                fapi2::P9_FAB_IOVALID_TL_NOT_TRAINED_ERR()
+                .set_TARGET(i_target)
+                .set_LOC_ENDP_TYPE(i_link_ctl.endp_type)
+                .set_LOC_ENDP_UNIT_ID(i_link_ctl.endp_unit_id)
+                .set_TL_FIR_REG(l_tl_fir_reg),
+                "Link TL training did not complete successfully!");
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
+/// @brief Compute single end link delay of individual link
+///
+/// @param[in]  i_target          Processor chip target
+/// @param[in]  i_link_ctl        X/A link control structure for link
+/// @param[out] o_link_delay      Link delay
+///
+/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+fapi2::ReturnCode p9_fab_iovalid_get_link_delay(
+    const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+    const p9_fbc_link_ctl_t& i_link_ctl,
+    uint32_t o_link_delay)
+{
+    FAPI_DBG("Start");
+    fapi2::buffer<uint64_t> l_link_delay_reg;
+    uint32_t l_sublink_delay[2];
+
+    // read link delay register, extract hi/lo delay values & return their average
+    FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.tl_link_delay_addr, l_link_delay_reg),
+             "Error from getScom (0x%.16llX)", i_link_ctl.tl_link_delay_addr);
+    FAPI_TRY(l_link_delay_reg.extractToRight(l_sublink_delay[0],
+             i_link_ctl.tl_link_delay_hi_start_bit,
+             TL_LINK_DELAY_FIELD_NUM_BITS),
+             "Error extracting link delay (hi>");
+    FAPI_TRY(l_link_delay_reg.extractToRight(l_sublink_delay[1],
+             i_link_ctl.tl_link_delay_lo_start_bit,
+             TL_LINK_DELAY_FIELD_NUM_BITS),
+             "Error extracting link delay (lo)");
+    o_link_delay = (l_sublink_delay[0] + l_sublink_delay[1]) / 2;
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
+/// @brief Get round trip training delays reported by both endpoints
+///        of a given link
 ///
-/// @brief Manipulate iovalid/FIR settings for a single fabric link
+/// @tparam T template parameter, defines endpoint type
+/// @param[in]  i_loc_target        Source side chip target
+/// @param[in]  i_loc_link_ctl      X/A link control structure for link local end
+/// @param[in]  i_rem_link_ctl      X/A link control structure for link remote end
+/// @param[out] o_agg_link_delay    Sum of local and remote end link delays
+///
+/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+template<fapi2::TargetType T>
+fapi2::ReturnCode p9_fab_iovalid_get_link_delays(
+    const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_loc_chip_target,
+    const p9_fbc_link_ctl_t& i_loc_link_ctl,
+    const p9_fbc_link_ctl_t& i_rem_link_ctl,
+    uint32_t& o_agg_link_delay)
+{
+    FAPI_DBG("Start");
+
+    bool l_found = false;
+    uint32_t l_loc_link_delay = 0xFFF;
+    uint32_t l_rem_link_delay = 0xFFF;
+    fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_rem_chip_target;
+
+    // use local end link control structure to find associated endpoint target
+    auto l_endp_targets = i_loc_chip_target.getChildren<T>();
+
+    for (auto l_iter = l_endp_targets.begin();
+         (l_iter != l_endp_targets.end()) && !l_found;
+         l_iter++)
+    {
+        uint8_t l_loc_endp_unit_id;
+        FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, *l_iter, l_loc_endp_unit_id),
+                 "Error from FAPI_ATTR_GET (ATTR_CHIP_UNIT_POS, local)");
+
+        if ((static_cast<fapi2::TargetType>(i_loc_link_ctl.endp_type) == T) &&
+            (i_loc_link_ctl.endp_unit_id == l_loc_endp_unit_id))
+        {
+            // associated endpoint target found, use getOtherEnd/getParent to reach chip
+            // target of connected chip
+            fapi2::Target<T> l_rem_endp_target;
+            fapi2::ReturnCode l_rc = l_iter->getOtherEnd(l_rem_endp_target);
+            FAPI_ASSERT(!l_rc,
+                        fapi2::P9_FAB_IOVALID_REM_ENDP_TARGET_ERR()
+                        .set_LOC_TARGET(i_loc_chip_target)
+                        .set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
+                        .set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id)
+                        .set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
+                        "Endpoint target at other end of link is invalid!");
+
+            l_rem_chip_target = l_rem_endp_target.template getParent<fapi2::TARGET_TYPE_PROC_CHIP>();
+            l_found = true;
+        }
+    }
+
+    FAPI_ASSERT(l_found,
+                fapi2::P9_FAB_IOVALID_LOC_ENDP_TARGET_ERR()
+                .set_LOC_TARGET(i_loc_chip_target)
+                .set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
+                .set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id),
+                "No matching local endpoint target found!");
+
+    // read link delay from local/remote chip targets
+    // link control structures provide register/bit offsets to collect
+    FAPI_TRY(p9_fab_iovalid_get_link_delay(
+                 i_loc_chip_target,
+                 i_loc_link_ctl,
+                 l_loc_link_delay),
+             "Error from p9_fab_iovalid_get_link_delay (local)");
+
+    FAPI_TRY(p9_fab_iovalid_get_link_delay(
+                 l_rem_chip_target,
+                 i_rem_link_ctl,
+                 l_rem_link_delay),
+             "Error from p9_fab_iovalid_get_link_delay (remote)");
+
+    o_agg_link_delay = l_loc_link_delay + l_rem_link_delay;
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
+
+///
+/// @brief Manipulate iovalid/FIR settings for a single fabric link (X/A)
+///
+/// @param[in] i_target        Reference to processor chip target
+/// @param[in] i_ctl           Reference to link control structure
+/// @param[in] i_set_not_clear Define iovalid operation (true=set, false=clear)
 ///
-/// @param[in] i_target Reference to processor chip target
-/// @param[in] i_ctl Reference to link control structure
-/// @param[op] i_set_not_clear Define iovalid operation (true=set, false=clear)
 /// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
 ///
 fapi2::ReturnCode
@@ -112,90 +294,130 @@ fapi_try_exit:
 }
 
 
-
+// NOTE: see doxygen comments in header
 fapi2::ReturnCode
 p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
-               const bool i_set_not_clear)
+               const bool i_set_not_clear,
+               const bool i_manage_electrical,
+               const bool i_manage_optical)
 {
     FAPI_INF("Start");
 
-    uint8_t l_x_en_attr[P9_FBC_UTILS_MAX_X_LINKS];
-    uint8_t l_a_en_attr[P9_FBC_UTILS_MAX_A_LINKS];
-    std::vector<std::pair<p9_fbc_link_t, uint8_t>> l_valid_links;
-    std::vector<p9_fbc_link_ctl_t> l_link_ctls(P9_FBC_LINK_CTL_ARR,
-            P9_FBC_LINK_CTL_ARR + (sizeof(P9_FBC_LINK_CTL_ARR) / sizeof(P9_FBC_LINK_CTL_ARR[0])));
-    bool l_ctl_match_found = false;
-
+    // logical link (X/A) configuration parameters
+    // arrays indexed by link ID on local end
+    // enable on local end
+    uint8_t l_x_en[P9_FBC_UTILS_MAX_X_LINKS];
+    uint8_t l_a_en[P9_FBC_UTILS_MAX_A_LINKS];
+    // link ID on remote end
+    uint8_t l_x_rem_link_id[P9_FBC_UTILS_MAX_X_LINKS];
+    uint8_t l_a_rem_link_id[P9_FBC_UTILS_MAX_A_LINKS];
+    // aggregate (local+remote) delays
+    uint32_t l_x_agg_link_delay[P9_FBC_UTILS_MAX_X_LINKS];
+    uint32_t l_a_agg_link_delay[P9_FBC_UTILS_MAX_A_LINKS];
 
-    // read X/A link enable attributes, extract set of valid links
-    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG,
-                           i_target,
-                           l_x_en_attr),
+    // seed arrays with attribute values
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG, i_target, l_x_en),
              "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG");
 
-    for (uint8_t x = 0; x < P9_FBC_UTILS_MAX_X_LINKS; x++)
-    {
-        if (l_x_en_attr[x])
-        {
-            FAPI_DBG("Adding link X%d", x);
-            l_valid_links.push_back(std::make_pair(XBUS, x));
-        }
-        else
-        {
-            FAPI_DBG("Skipping link X%d", x);
-        }
-    }
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_LINK_ID, i_target, l_x_rem_link_id),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_LINK_ID");
+
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_LINK_DELAY, i_target, l_x_agg_link_delay),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_LINK_DELAY");
 
-    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG,
-                           i_target,
-                           l_a_en_attr),
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG, i_target, l_a_en),
              "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG");
 
-    for (uint8_t a = 0; a < P9_FBC_UTILS_MAX_A_LINKS; a++)
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_LINK_ID, i_target, l_a_rem_link_id),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_LINK_ID");
+
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_LINK_DELAY, i_target, l_a_agg_link_delay),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_LINK_DELAY");
+
+    for (uint8_t l_link_id = 0; l_link_id < P9_FBC_UTILS_MAX_X_LINKS; l_link_id++)
     {
-        if (l_a_en_attr[a])
+        if (l_x_en[l_link_id])
         {
-            FAPI_DBG("Adding link A%d", a);
-            l_valid_links.push_back(std::make_pair(ABUS, a));
+            if ((i_manage_electrical &&
+                 (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == ELECTRICAL)) ||
+                (i_manage_optical &&
+                 (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL)))
+            {
+                FAPI_DBG("Updating link X%d", l_link_id);
+                FAPI_TRY(p9_fab_iovalid_update_link(i_target,
+                                                    P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
+                                                    i_set_not_clear),
+                         "Error from p9_fab_iovalid_update_link (X)");
+
+                if (i_set_not_clear)
+                {
+                    FAPI_DBG("Collecting link delay counter values");
+
+                    if (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == ELECTRICAL)
+                    {
+                        FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_XBUS>(
+                                     i_target,
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
+                                     l_x_agg_link_delay[l_link_id]),
+                                 "Error from p9_fab_iovalid_get_link_delays (X, electrical)");
+                    }
+                    else
+                    {
+                        FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_OBUS>(
+                                     i_target,
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
+                                     l_x_agg_link_delay[l_link_id]),
+                                 "Error from p9_fab_iovalid_get_link_delays (X, optical)");
+                    }
+                }
+            }
         }
         else
         {
-            FAPI_DBG("Skipping link A%d", a);
+            FAPI_DBG("Skipping link X%d", l_link_id);
         }
     }
 
-    // for each valid link, search vector table & call link update routine
-    for (auto l_link_iter = l_valid_links.begin(); l_link_iter != l_valid_links.end(); l_link_iter++)
+    for (uint8_t l_link_id = 0; l_link_id < P9_FBC_UTILS_MAX_A_LINKS; l_link_id++)
     {
-        FAPI_DBG("Processing %s%d (action = %s)",
-                 (l_link_iter->first == XBUS) ? ("X") : ("A)"),
-                 l_link_iter->second,
-                 (i_set_not_clear) ? ("set") : ("clear"));
-
-        l_ctl_match_found = false;
-
-        for (auto l_link_ctl_iter = l_link_ctls.begin();
-             (l_link_ctl_iter != l_link_ctls.end()) && (!l_ctl_match_found);
-             l_link_ctl_iter++)
+        if (l_a_en[l_link_id])
         {
-            if ((l_link_ctl_iter->link_type == l_link_iter->first) &&
-                (l_link_ctl_iter->link_id == l_link_iter->second))
+            if (i_manage_optical &&
+                (P9_FBC_ABUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL))
             {
-                l_ctl_match_found = true;
+                FAPI_DBG("Updating link A%d", l_link_id);
                 FAPI_TRY(p9_fab_iovalid_update_link(i_target,
-                                                    *l_link_ctl_iter,
+                                                    P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
                                                     i_set_not_clear),
-                         "Error from p9_fab_iovalid_update_link");
+                         "Error from p9_fab_iovalid_update_link (A)");
+
+                if (i_set_not_clear)
+                {
+                    FAPI_DBG("Collecting link delay counter values");
+                    FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_OBUS>(
+                                 i_target,
+                                 P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
+                                 P9_FBC_ABUS_LINK_CTL_ARR[l_a_rem_link_id[l_link_id]],
+                                 l_a_agg_link_delay[l_link_id]),
+                             "Error from p9_fab_iovalid_get_link_delays (A)");
+                }
             }
         }
-
-        FAPI_ASSERT(l_ctl_match_found,
-                    fapi2::P9_FAB_IOVALID_TABLE_ERR().set_TARGET(i_target).
-                    set_LINK(l_link_iter->first).
-                    set_LINK_ID(l_link_iter->second),
-                    "No match found for link");
+        else
+        {
+            FAPI_DBG("Skipping link A%d", l_link_id);
+        }
     }
 
+    // update link delay attributes
+    FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_X_LINK_DELAY, i_target, l_x_agg_link_delay),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_LINK_DELAY");
+
+    FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_A_LINK_DELAY, i_target, l_a_agg_link_delay),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_LINK_DELAY");
+
 fapi_try_exit:
     FAPI_INF("End");
     return fapi2::current_err;