support IO reconfig loop for OBUS DL link training failures

- p9c DD1.1+ only, DD1.0 not supported

- FW to trigger reconfig loop back to step 0 from
  sys_proc_fab_iovalid if:
  1) sys_proc_fab_iovalid rc = FAPI2_RC_SUCCESS
     -- AND --
  2) new output o_obus_dl_rcs (vector of P9_FAB_IOVALID_DL_NOT_TRAINED_ERR rc
     objects identifying links which failed on this chip) has entries

- attribute changes:
  ATTR_LINK_TRAIN -- remove platinit tag, attr should init to zero (both even
                     and odd), and reconfig loop will adjust value as we go

CMVC-Prereq:1057645
Change-Id: I95eebd2b893db6d2511aae40798c0a4e049835d6
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59022
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59039
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

10 files changed, 331 insertions, 54 deletions

diff --git a/src/import/chips/p9/common/include/p9_obus_scom_addresses_fixes.H b/src/import/chips/p9/common/include/p9_obus_scom_addresses_fixes.H
index 664d93d3b..eeef5f374 100644
--- a/src/import/chips/p9/common/include/p9_obus_scom_addresses_fixes.H
+++ b/src/import/chips/p9/common/include/p9_obus_scom_addresses_fixes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -53,11 +53,15 @@ static const uint64_t SH_UNT_OBUS_2 = 515;
 
 REG64( OBUS_1_LL1_IOOL_CONTROL,
        RULL(0x0A01080B), SH_UNT_OBUS_1, SH_ACS_SCOM );
+REG64( OBUS_1_LL1_IOOL_DLL_STATUS,
+       RULL(0x0A010828), SH_UNT_OBUS_1, SH_ACS_SCOM );
 REG64( OBUS_1_LL1_LL1_LL1_PB_IOOL_FIR_REG,
        RULL(0x0A010800), SH_UNT_OBUS_2, SH_ACS_SCOM );
 
 REG64( OBUS_2_LL2_IOOL_CONTROL,
        RULL(0x0B01080B), SH_UNT_OBUS_2, SH_ACS_SCOM );
+REG64( OBUS_2_LL2_IOOL_DLL_STATUS,
+       RULL(0x0B010828), SH_UNT_OBUS_2, SH_ACS_SCOM );
 REG64( OBUS_2_LL2_LL2_LL2_PB_IOOL_FIR_REG,
        RULL(0x0B010800), SH_UNT_OBUS_2, SH_ACS_SCOM );
 
diff --git a/src/import/chips/p9/procedures/hwp/io/p9_io_obus_dccal.C b/src/import/chips/p9/procedures/hwp/io/p9_io_obus_dccal.C
index 8c74590eb..9cbced143 100644
--- a/src/import/chips/p9/procedures/hwp/io/p9_io_obus_dccal.C
+++ b/src/import/chips/p9/procedures/hwp/io/p9_io_obus_dccal.C
@@ -667,7 +667,7 @@ fapi2::ReturnCode p9_io_obus_dccal( const OBUS_TGT i_tgt, const uint32_t i_lane_
         goto fapi_try_exit;
     }
 
-    FAPI_TRY( FAPI_ATTR_GET( fapi2::ATTR_IO_OBUS_DCCAL_FLAGS, i_tgt, dccal_flags ) );
+    FAPI_TRY( FAPI_ATTR_SET( fapi2::ATTR_IO_OBUS_DCCAL_FLAGS, i_tgt, dccal_flags ) );
 
     // Power up Clock Distribution & Lanes
     FAPI_TRY( obus_powerup( i_tgt, i_lane_vector ) );
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
index ab5df082b..6bd4416ab 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
@@ -42,6 +42,8 @@
 //------------------------------------------------------------------------------
 #include <p9_fab_iovalid.H>
 #include <p9_fbc_smp_utils.H>
+#include <p9_obus_scom_addresses.H>
+#include <p9_obus_scom_addresses_fld.H>
 #include <p9_misc_scom_addresses.H>
 #include <p9_misc_scom_addresses_fld.H>
 
@@ -464,13 +466,20 @@ fapi2::ReturnCode p9_fab_iovalid_link_validate(
     FAPI_DBG("Start");
     fapi2::buffer<uint64_t> l_dl_fir_reg;
     fapi2::buffer<uint64_t> l_tl_fir_reg;
+    fapi2::buffer<uint64_t> l_dl_status_reg;
     fapi2::Target<T> l_loc_endp_target;
     fapi2::Target<T> l_rem_endp_target;
     fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_rem_chip_target;
     fapi2::ATTR_LINK_TRAIN_Type l_loc_link_train;
+    fapi2::ATTR_LINK_TRAIN_Type l_loc_link_train_next;
     bool l_dl_trained = false;
-    bool l_tl_trained = false;
+    uint8_t l_dl_status_even = 0;
+    bool l_dl_fail_even = false;
+    uint8_t l_dl_status_odd = 0;
+    bool l_dl_fail_odd = false;
+    uint8_t l_tl_trained = 0;
     uint32_t l_poll_loops = DL_MAX_POLL_LOOPS;
+    char l_target_str[fapi2::MAX_ECMD_STRING_LEN];
 
     // obtain link endpoints for FFDC
     FAPI_TRY(p9_fab_iovalid_get_link_endpoints(i_target,
@@ -481,52 +490,128 @@ fapi2::ReturnCode p9_fab_iovalid_link_validate(
              l_rem_chip_target),
              "Error from p9_fab_iovalid_get_link_endpoints");
 
+    fapi2::toString(l_loc_endp_target, l_target_str, sizeof(l_target_str));
+
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_LINK_TRAIN,
                            l_loc_endp_target,
                            l_loc_link_train),
              "Error from FAPI_ATTR_GET (ATTR_LINK_TRAIN)");
+    l_loc_link_train_next = l_loc_link_train;
 
+    // poll for DL trained indications
     do
     {
         // validate DL training state
         FAPI_TRY(fapi2::getScom(i_target, i_loc_link_ctl.dl_fir_addr, l_dl_fir_reg),
                  "Error from getScom (0x%.16llX)", i_loc_link_ctl.dl_fir_addr);
 
+        FAPI_TRY(fapi2::getScom(i_target, i_loc_link_ctl.dl_status_addr, l_dl_status_reg),
+                 "Error from getScom (0x%.16llX)", i_loc_link_ctl.dl_status_addr);
+
+        l_dl_status_reg.extractToRight<XBUS_LL0_IOEL_DLL_STATUS_LINK0_CURRENT_STATE,
+                                       XBUS_LL0_IOEL_DLL_STATUS_LINK0_CURRENT_STATE_LEN>
+                                       (l_dl_status_even);
+
+        l_dl_status_reg.extractToRight<XBUS_LL0_IOEL_DLL_STATUS_LINK1_CURRENT_STATE,
+                                       XBUS_LL0_IOEL_DLL_STATUS_LINK1_CURRENT_STATE_LEN>
+                                       (l_dl_status_odd);
+
         if (l_loc_link_train == fapi2::ENUM_ATTR_LINK_TRAIN_BOTH)
         {
-            l_dl_trained = l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>() &&
-                           l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>();
+            l_dl_trained   = l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>() &&
+                             l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>();
+
+            l_dl_fail_even = !((l_dl_status_even == 0x8) && ((l_dl_status_odd  >= 0xB) && (l_dl_status_odd  <= 0xE)));
+            l_dl_fail_odd  = !((l_dl_status_odd  == 0x8) && ((l_dl_status_even >= 0xB) && (l_dl_status_even <= 0xE)));
         }
         else if (l_loc_link_train == fapi2::ENUM_ATTR_LINK_TRAIN_EVEN_ONLY)
         {
-            l_dl_trained = l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>();
+            l_dl_trained   = l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>();
+            l_dl_fail_even = !l_dl_trained;
+            l_dl_fail_odd  = true;
         }
         else
         {
-            l_dl_trained = l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>();
+            l_dl_trained   = l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>();
+            l_dl_fail_even = true;
+            l_dl_fail_odd  = !l_dl_trained;
         }
 
         if (!l_dl_trained)
         {
             FAPI_TRY(fapi2::delay(DL_POLL_HW_DELAY_NS, DL_POLL_SIM_CYCLES), "fapiDelay error");
-
         }
 
         l_poll_loops--;
     }
     while (l_poll_loops > 0 && !l_dl_trained);
 
-    FAPI_ASSERT(l_dl_trained,
-                fapi2::P9_FAB_IOVALID_DL_NOT_TRAINED_ERR()
-                .set_TARGET(i_target)
-                .set_LOC_ENDP_TARGET(l_loc_endp_target)
-                .set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
-                .set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id)
-                .set_LOC_LINK_TRAIN(l_loc_link_train)
-                .set_REM_ENDP_TARGET(l_rem_endp_target)
-                .set_REM_ENDP_TYPE(i_rem_link_ctl.endp_type)
-                .set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
-                "Link DL training did not complete successfully!");
+    if (!l_dl_trained)
+    {
+        FAPI_ERR("Error in DL training for %s (ATTR_LINK_TRAIN: 0x%x, DL training failed: %d / %d)",
+                 l_target_str,
+                 l_loc_link_train,
+                 l_dl_fail_even,
+                 l_dl_fail_odd);
+
+        if ((l_loc_link_train == fapi2::ENUM_ATTR_LINK_TRAIN_BOTH) &&
+            ((l_dl_fail_even != true) || (l_dl_fail_odd != true)) &&
+            (T == fapi2::TARGET_TYPE_OBUS))
+        {
+            // setup to retrain half link only
+            l_loc_link_train_next = (l_dl_fail_even) ?
+                                    (fapi2::ENUM_ATTR_LINK_TRAIN_ODD_ONLY) :
+                                    (fapi2::ENUM_ATTR_LINK_TRAIN_EVEN_ONLY);
+
+            FAPI_DBG("Setting up to retrain with ATTR_LINK_TRAIN: 0x%x",
+                     l_loc_link_train_next);
+        }
+        else
+        {
+            l_loc_link_train_next = fapi2::ENUM_ATTR_LINK_TRAIN_NONE;
+        }
+
+        FAPI_INF("Resetting ATTR_LINK_TRAIN: 0x%X for %s",
+                 l_loc_link_train_next,
+                 l_target_str);
+
+        FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_LINK_TRAIN,
+                               l_loc_endp_target,
+                               l_loc_link_train_next),
+                 "Error from FAPI_ATTR_SET (ATTR_LINK_TRAIN)");
+
+        // if nothing is left to run on, emit RC with callout/deconfig on endpoints
+        // (no retraining for this bus)
+        FAPI_ASSERT(l_loc_link_train_next != fapi2::ENUM_ATTR_LINK_TRAIN_NONE,
+                    fapi2::P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_NONE_ERR()
+                    .set_TARGET(i_target)
+                    .set_LOC_ENDP_TARGET(l_loc_endp_target)
+                    .set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
+                    .set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id)
+                    .set_LOC_LINK_TRAIN(l_loc_link_train)
+                    .set_LOC_LINK_TRAIN_NEXT(l_loc_link_train_next)
+                    .set_LOC_LINK_FAILED0(l_dl_fail_even)
+                    .set_LOC_LINK_FAILED1(l_dl_fail_odd)
+                    .set_REM_ENDP_TARGET(l_rem_endp_target)
+                    .set_REM_ENDP_TYPE(i_rem_link_ctl.endp_type)
+                    .set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
+                    "Link DL training did not complete successfully!");
+        // else, emit RC with no deconfig (and attempt retraining on half link)
+        FAPI_ASSERT(false,
+                    fapi2::P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_HALF_ERR()
+                    .set_TARGET(i_target)
+                    .set_LOC_ENDP_TARGET(l_loc_endp_target)
+                    .set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
+                    .set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id)
+                    .set_LOC_LINK_TRAIN(l_loc_link_train)
+                    .set_LOC_LINK_TRAIN_NEXT(l_loc_link_train_next)
+                    .set_LOC_LINK_FAILED0(l_dl_fail_even)
+                    .set_LOC_LINK_FAILED1(l_dl_fail_odd)
+                    .set_REM_ENDP_TARGET(l_rem_endp_target)
+                    .set_REM_ENDP_TYPE(i_rem_link_ctl.endp_type)
+                    .set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
+                    "Link DL training did not complete successfully!");
+    }
 
     // validate TL training state
     FAPI_TRY(fapi2::getScom(i_target, i_loc_link_ctl.tl_fir_addr, l_tl_fir_reg),
@@ -556,11 +641,14 @@ fapi2::ReturnCode p9_fab_iovalid_link_validate(
                 .set_REM_ENDP_TARGET(l_rem_endp_target)
                 .set_REM_ENDP_TYPE(i_rem_link_ctl.endp_type)
                 .set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
-                "Link TL training did not complete successfully!");
+                "Error in TL training for %s (ATTR_LINK_TRAIN: 0x%X)",
+                l_target_str,
+                l_loc_link_train);
 
 fapi_try_exit:
 
-    if (fapi2::current_err == (fapi2::ReturnCode) fapi2::RC_P9_FAB_IOVALID_DL_NOT_TRAINED_ERR)
+    if ((fapi2::current_err == (fapi2::ReturnCode) fapi2::RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_NONE_ERR) ||
+        (fapi2::current_err == (fapi2::ReturnCode) fapi2::RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_HALF_ERR))
     {
         p9_fab_iovalid_append_dl_ffdc<T>(i_target,
                                          i_loc_link_ctl,
@@ -584,6 +672,53 @@ fapi_try_exit:
 }
 
 
+/// @brief Validate DL/TL link layers are trained
+///
+/// @param[in]  i_target          Processor chip target
+/// @param[in]  i_loc_link_ctl    X/A link control structure for link local end
+/// @param[in]  i_rem_link_ctl    X/A link control structure for link remote end
+/// @param[out] o_retrain         Indication that DL link training should be
+///                               re-attempted
+/// @param[out] o_rcs             Vector of return code objects, to append
+///                               in case of reported DL training failure
+///
+/// @return fapi2::ReturnCode
+template<fapi2::TargetType T>
+fapi2::ReturnCode p9_fab_iovalid_link_validate_wrap(
+    const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+    const p9_fbc_link_ctl_t& i_loc_link_ctl,
+    const p9_fbc_link_ctl_t& i_rem_link_ctl,
+    bool& o_retrain,
+    std::vector<fapi2::ReturnCode>& o_rcs)
+{
+    fapi2::ReturnCode l_rc;
+    l_rc = p9_fab_iovalid_link_validate<T>(
+               i_target,
+               i_loc_link_ctl,
+               i_rem_link_ctl);
+
+    if (((l_rc == (fapi2::ReturnCode) fapi2::RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_NONE_ERR) ||
+         (l_rc == (fapi2::ReturnCode) fapi2::RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_HALF_ERR)) &&
+        (T == fapi2::TARGET_TYPE_OBUS))
+    {
+        o_retrain = true;
+        o_rcs.push_back(l_rc);
+        return fapi2::FAPI2_RC_SUCCESS;
+    }
+    else
+    {
+        o_retrain = false;
+
+        if (l_rc != fapi2::FAPI2_RC_SUCCESS)
+        {
+            FAPI_ERR("Error from p9_fab_iovalid_link_validate_wrap");
+        }
+
+        return l_rc;
+    }
+}
+
+
 /// @brief Compute single end link delay of individual link
 ///
 /// @param[in]  i_target          Processor chip target
@@ -817,7 +952,8 @@ fapi2::ReturnCode
 p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                const bool i_set_not_clear,
                const bool i_manage_electrical,
-               const bool i_manage_optical)
+               const bool i_manage_optical,
+               std::vector<fapi2::ReturnCode>& o_obus_dl_fail_rcs)
 {
     FAPI_INF("Start");
 
@@ -865,7 +1001,39 @@ p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                 (i_manage_optical &&
                  (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL)))
             {
+                bool l_link_needs_retraining = false;
+
                 FAPI_DBG("Updating link X%d", l_link_id);
+
+                if (i_set_not_clear)
+                {
+                    if (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == ELECTRICAL)
+                    {
+                        FAPI_TRY(p9_fab_iovalid_link_validate_wrap<fapi2::TARGET_TYPE_XBUS>(
+                                     i_target,
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
+                                     l_link_needs_retraining,
+                                     o_obus_dl_fail_rcs),
+                                 "Error from p9_fab_iovalid_link_validate_wrap (X, electrical)");
+                    }
+                    else
+                    {
+                        FAPI_TRY(p9_fab_iovalid_link_validate_wrap<fapi2::TARGET_TYPE_OBUS>(
+                                     i_target,
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
+                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
+                                     l_link_needs_retraining,
+                                     o_obus_dl_fail_rcs),
+                                 "Error from p9_fab_iovalid_link_validate_wrap (X, optical)");
+                    }
+                }
+
+                if (l_link_needs_retraining)
+                {
+                    continue;
+                }
+
                 FAPI_TRY(p9_fab_iovalid_update_link(i_target,
                                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
                                                     i_set_not_clear,
@@ -884,12 +1052,6 @@ p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                                      P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
                                      l_x_agg_link_delay[l_link_id]),
                                  "Error from p9_fab_iovalid_get_link_delays (X, electrical)");
-
-                        FAPI_TRY(p9_fab_iovalid_link_validate<fapi2::TARGET_TYPE_XBUS>(
-                                     i_target,
-                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
-                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]]),
-                                 "Error from p9_fab_iovalid_link_validate (X, electrical)");
                     }
                     else
                     {
@@ -899,12 +1061,6 @@ p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                                      P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
                                      l_x_agg_link_delay[l_link_id]),
                                  "Error from p9_fab_iovalid_get_link_delays (X, optical)");
-
-                        FAPI_TRY(p9_fab_iovalid_link_validate<fapi2::TARGET_TYPE_OBUS>(
-                                     i_target,
-                                     P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
-                                     P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]]),
-                                 "Error from p9_fab_iovalid_link_validate (X, optical)");
                     }
                 }
             }
@@ -922,7 +1078,25 @@ p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
             if (i_manage_optical &&
                 (P9_FBC_ABUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL))
             {
+                bool l_link_needs_retraining = false;
                 FAPI_DBG("Updating link A%d", l_link_id);
+
+                if (i_set_not_clear)
+                {
+                    FAPI_TRY(p9_fab_iovalid_link_validate_wrap<fapi2::TARGET_TYPE_OBUS>(
+                                 i_target,
+                                 P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
+                                 P9_FBC_ABUS_LINK_CTL_ARR[l_a_rem_link_id[l_link_id]],
+                                 l_link_needs_retraining,
+                                 o_obus_dl_fail_rcs),
+                             "Error from p9_fab_iovalid_link_validate_wrap (A)");
+                }
+
+                if (l_link_needs_retraining)
+                {
+                    continue;
+                }
+
                 FAPI_TRY(p9_fab_iovalid_update_link(i_target,
                                                     P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
                                                     i_set_not_clear,
@@ -938,12 +1112,6 @@ p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                                  P9_FBC_ABUS_LINK_CTL_ARR[l_a_rem_link_id[l_link_id]],
                                  l_a_agg_link_delay[l_link_id]),
                              "Error from p9_fab_iovalid_get_link_delays (A)");
-
-                    FAPI_TRY(p9_fab_iovalid_link_validate<fapi2::TARGET_TYPE_OBUS>(
-                                 i_target,
-                                 P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
-                                 P9_FBC_ABUS_LINK_CTL_ARR[l_a_rem_link_id[l_link_id]]),
-                             "Error from p9_fab_iovalid_link_validate (A)");
                 }
             }
         }
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
index d68c5bcf5..d71b41822 100755
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -66,7 +66,8 @@ typedef fapi2::ReturnCode (*p9_fab_iovalid_FP_t) (
     const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>&,
     const bool,
     const bool,
-    const bool);
+    const bool,
+    std::vector<fapi2::ReturnCode>&);
 
 //------------------------------------------------------------------------------
 // Function prototypes
@@ -82,13 +83,16 @@ extern "C"
 /// @param[in] i_set_not_clear       Define iovalid operation (true=set, false=clear)
 /// @param[in] i_manage_electrical   Manage electrical links?
 /// @param[in] i_manage_optical      Manage optical links?
+/// @param[out] o_obus_dl_fail_rcs   Vector of return code objects, to append
+///                                  in case of OBUS DL training failures
 ///
 /// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
 ///
     fapi2::ReturnCode p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                                      const bool i_set_not_clear,
                                      const bool i_manage_electrical,
-                                     const bool i_manage_optical);
+                                     const bool i_manage_optical,
+                                     std::vector<fapi2::ReturnCode>& o_obus_dl_fail_rcs);
 
 } // extern "C"
 
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config.C b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config.C
index 208f9c767..fd41861f4 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config.C
@@ -505,6 +505,46 @@ fapi_try_exit:
 }
 
 
+///
+/// @brief Reset attributes related to link usage
+///
+/// @param[in]     i_target                System target
+///
+/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+///
+fapi2::ReturnCode
+p9_fbc_eff_config_reset_attrs(
+    const fapi2::Target<fapi2::TARGET_TYPE_SYSTEM>& i_target)
+{
+    FAPI_DBG("Start");
+
+    fapi2::ATTR_PROC_FABRIC_LINK_ACTIVE_Type l_link_active = fapi2::ENUM_ATTR_PROC_FABRIC_LINK_ACTIVE_FALSE;
+
+    for (auto l_proc_target : i_target.getChildren<fapi2::TARGET_TYPE_PROC_CHIP>())
+    {
+        for (auto l_obus_chiplet_target : l_proc_target.getChildren<fapi2::TARGET_TYPE_OBUS>())
+        {
+            FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_LINK_ACTIVE,
+                                   l_obus_chiplet_target,
+                                   l_link_active),
+                     "Error from FAPI_ATTR_SET (ATTR_PROC_FABRIC_LINK_ACTIVE (OBUS)");
+        }
+
+        for (auto l_xbus_chiplet_target : l_proc_target.getChildren<fapi2::TARGET_TYPE_XBUS>())
+        {
+            FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_LINK_ACTIVE,
+                                   l_xbus_chiplet_target,
+                                   l_link_active),
+                     "Error from FAPI_ATTR_SET (ATTR_PROC_FABRIC_LINK_ACTIVE (XBUS)");
+        }
+    }
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
 // NOTE: see doxygen comments in header
 fapi2::ReturnCode
 p9_fbc_eff_config()
@@ -533,6 +573,10 @@ p9_fbc_eff_config()
                  l_freq_core_ceiling),
              "Error from p9_fbc_eff_config_calc_epsilons");
 
+    FAPI_TRY(p9_fbc_eff_config_reset_attrs(
+                 FAPI_SYSTEM),
+             "Error from p9_fbc_eff_config_reset_attrs");
+
 fapi_try_exit:
     FAPI_DBG("End");
     return fapi2::current_err;
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_aggregate.C b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_aggregate.C
index 30b9efd09..29983dd9c 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_aggregate.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_aggregate.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -110,7 +110,7 @@ fapi2::ReturnCode p9_fbc_eff_config_aggregate_link_setup(
             // flip default value for link address disable
             for (uint8_t l_loc_link_id = 0; l_loc_link_id < i_max_links; l_loc_link_id++)
             {
-                o_addr_dis[l_loc_link_id] = i_en[l_loc_link_id];
+                o_addr_dis[l_loc_link_id] = (i_en[l_loc_link_id]) ? (1) : (0);
             }
 
             // scan link delays for smallest value
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_links.C b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_links.C
index 83a7b7ee9..5e2bb5a4c 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_links.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_eff_config_links.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -80,10 +80,14 @@ fapi2::ReturnCode p9_fbc_eff_config_links_query_link_en(
     {
         o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_EVEN_ONLY;
     }
-    else
+    else if (l_link_train == fapi2::ENUM_ATTR_LINK_TRAIN_ODD_ONLY)
     {
         o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_ODD_ONLY;
     }
+    else
+    {
+        o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_FALSE;
+    }
 
 fapi_try_exit:
     FAPI_DBG("End, o_link_is_enabled: 0x%x", o_link_is_enabled);
@@ -105,7 +109,7 @@ fapi2::ReturnCode p9_fbc_eff_config_links_query_link_en(
 
     if (l_link_config_mode != fapi2::ENUM_ATTR_OPTICS_CONFIG_MODE_SMP)
     {
-        o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG_FALSE;
+        o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_FALSE;
     }
     else
     {
@@ -123,10 +127,14 @@ fapi2::ReturnCode p9_fbc_eff_config_links_query_link_en(
         {
             o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_EVEN_ONLY;
         }
-        else
+        else if (l_link_train == fapi2::ENUM_ATTR_LINK_TRAIN_ODD_ONLY)
         {
             o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_ODD_ONLY;
         }
+        else
+        {
+            o_link_is_enabled = fapi2::ENUM_ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG_FALSE;
+        }
     }
 
 fapi_try_exit:
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_smp_utils.H b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_smp_utils.H
index ea4fc0032..8f2a1849b 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_smp_utils.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_smp_utils.H
@@ -79,6 +79,7 @@ struct p9_fbc_link_ctl_t
     // DL layer SCOM addresses
     uint64_t      dl_fir_addr;
     uint64_t      dl_control_addr;
+    uint64_t      dl_status_addr;
     // TL layer SCOM addresses
     uint64_t      tl_fir_addr;
     uint8_t       tl_fir_trained_field_start_bit;
@@ -105,6 +106,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X0_FIR_ERR,
         XBUS_LL0_LL0_LL0_IOEL_FIR_REG,
         XBUS_0_LL0_IOEL_CONTROL,
+        XBUS_LL0_IOEL_DLL_STATUS,
         PU_PB_IOE_FIR_REG,
         PU_PB_IOE_FIR_REG_FMR00_TRAINED,
         PU_PB_ELINK_DLY_0123_REG,
@@ -120,6 +122,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X1_FIR_ERR,
         XBUS_1_LL1_LL1_LL1_IOEL_FIR_REG,
         XBUS_1_LL1_IOEL_CONTROL,
+        XBUS_1_LL1_IOEL_DLL_STATUS,
         PU_PB_IOE_FIR_REG,
         PU_PB_IOE_FIR_REG_FMR02_TRAINED,
         PU_PB_ELINK_DLY_0123_REG,
@@ -135,6 +138,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X2_FIR_ERR,
         XBUS_2_LL2_LL2_LL2_IOEL_FIR_REG,
         XBUS_2_LL2_IOEL_CONTROL,
+        XBUS_2_LL2_IOEL_DLL_STATUS,
         PU_PB_IOE_FIR_REG,
         PU_PB_IOE_FIR_REG_FMR04_TRAINED,
         PU_PB_ELINK_DLY_45_REG,
@@ -150,6 +154,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X3_FIR_ERR,
         OBUS_LL0_LL0_LL0_PB_IOOL_FIR_REG,
         OBUS_0_LL0_IOOL_CONTROL,
+        OBUS_0_LL0_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR00_TRAINED,
         PU_IOE_PB_OLINK_DLY_0123_REG,
@@ -165,6 +170,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X4_FIR_ERR,
         OBUS_1_LL1_LL1_LL1_PB_IOOL_FIR_REG,
         OBUS_1_LL1_IOOL_CONTROL,
+        OBUS_1_LL1_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR02_TRAINED,
         PU_IOE_PB_OLINK_DLY_0123_REG,
@@ -180,6 +186,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X5_FIR_ERR,
         OBUS_2_LL2_LL2_LL2_PB_IOOL_FIR_REG,
         OBUS_2_LL2_IOOL_CONTROL,
+        OBUS_2_LL2_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR04_TRAINED,
         PU_IOE_PB_OLINK_DLY_4567_REG,
@@ -195,6 +202,7 @@ const p9_fbc_link_ctl_t P9_FBC_XBUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X6_FIR_ERR,
         OBUS_3_LL3_LL3_LL3_PB_IOOL_FIR_REG,
         OBUS_3_LL3_IOOL_CONTROL,
+        OBUS_3_LL3_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR06_TRAINED,
         PU_IOE_PB_OLINK_DLY_4567_REG,
@@ -214,6 +222,7 @@ const p9_fbc_link_ctl_t P9_FBC_ABUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_A_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X3_FIR_ERR,
         OBUS_LL0_LL0_LL0_PB_IOOL_FIR_REG,
         OBUS_0_LL0_IOOL_CONTROL,
+        OBUS_0_LL0_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR00_TRAINED,
         PU_IOE_PB_OLINK_DLY_0123_REG,
@@ -229,6 +238,7 @@ const p9_fbc_link_ctl_t P9_FBC_ABUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_A_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X4_FIR_ERR,
         OBUS_1_LL1_LL1_LL1_PB_IOOL_FIR_REG,
         OBUS_1_LL1_IOOL_CONTROL,
+        OBUS_1_LL1_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR02_TRAINED,
         PU_IOE_PB_OLINK_DLY_0123_REG,
@@ -244,6 +254,7 @@ const p9_fbc_link_ctl_t P9_FBC_ABUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_A_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X5_FIR_ERR,
         OBUS_2_LL2_LL2_LL2_PB_IOOL_FIR_REG,
         OBUS_2_LL2_IOOL_CONTROL,
+        OBUS_2_LL2_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR04_TRAINED,
         PU_IOE_PB_OLINK_DLY_4567_REG,
@@ -259,6 +270,7 @@ const p9_fbc_link_ctl_t P9_FBC_ABUS_LINK_CTL_ARR[P9_FBC_UTILS_MAX_A_LINKS] =
         PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X6_FIR_ERR,
         OBUS_3_LL3_LL3_LL3_PB_IOOL_FIR_REG,
         OBUS_3_LL3_IOOL_CONTROL,
+        OBUS_3_LL3_IOOL_DLL_STATUS,
         PU_IOE_PB_IOO_FIR_REG,
         PU_IOE_PB_IOO_FIR_REG_FMR06_TRAINED,
         PU_IOE_PB_OLINK_DLY_4567_REG,
diff --git a/src/import/chips/p9/procedures/xml/attribute_info/nest_attributes.xml b/src/import/chips/p9/procedures/xml/attribute_info/nest_attributes.xml
index db13c9ede..f52d98822 100644
--- a/src/import/chips/p9/procedures/xml/attribute_info/nest_attributes.xml
+++ b/src/import/chips/p9/procedures/xml/attribute_info/nest_attributes.xml
@@ -533,10 +533,11 @@
   <enum>
     BOTH = 0x0,
     EVEN_ONLY = 0x1,
-    ODD_ONLY = 0x2
+    ODD_ONLY = 0x2,
+    NONE = 0x3
   </enum>
   <initToZero/>
-  <platInit/>
+  <writeable/>
 </attribute>
 <!-- ********************************************************************** -->
 <attribute>
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_fab_iovalid_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_fab_iovalid_errors.xml
index 241cc4743..643759530 100644
--- a/src/import/chips/p9/procedures/xml/error_info/p9_fab_iovalid_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_fab_iovalid_errors.xml
@@ -5,7 +5,7 @@
 <!--                                                                        -->
 <!-- OpenPOWER HostBoot Project                                             -->
 <!--                                                                        -->
-<!-- Contributors Listed Below - COPYRIGHT 2015,2017                        -->
+<!-- Contributors Listed Below - COPYRIGHT 2015,2018                        -->
 <!-- [+] International Business Machines Corp.                              -->
 <!--                                                                        -->
 <!--                                                                        -->
@@ -103,16 +103,20 @@
   </hwpError>
   <!-- ******************************************************************** -->
   <hwpError>
-    <rc>RC_P9_FAB_IOVALID_DL_NOT_TRAINED_ERR</rc>
+    <rc>RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_NONE_ERR</rc>
     <description>
       Procedure: p9_fab_iovalid
-      Link DL training did not complete successfully
+      Link DL training did not complete successfully,
+      both halves of link pair are incapable of training.
     </description>
     <ffdc>TARGET</ffdc>
     <ffdc>LOC_ENDP_TARGET</ffdc>
     <ffdc>LOC_ENDP_TYPE</ffdc>
     <ffdc>LOC_ENDP_UNIT_ID</ffdc>
     <ffdc>LOC_LINK_TRAIN</ffdc>
+    <ffdc>LOC_LINK_TRAIN_NEXT</ffdc>
+    <ffdc>LOC_LINK_FAILED0</ffdc>
+    <ffdc>LOC_LINK_FAILED1</ffdc>
     <ffdc>REM_ENDP_TARGET</ffdc>
     <ffdc>REM_ENDP_TYPE</ffdc>
     <ffdc>REM_ENDP_UNIT_ID</ffdc>
@@ -143,6 +147,38 @@
   </hwpError>
   <!-- ******************************************************************** -->
   <hwpError>
+    <rc>RC_P9_FAB_IOVALID_DL_NOT_TRAINED_RETRAIN_HALF_ERR</rc>
+    <description>
+      Procedure: p9_fab_iovalid
+      Link DL training did not complete successfully,
+      attempt retraining on half link basis.
+    </description>
+    <ffdc>TARGET</ffdc>
+    <ffdc>LOC_ENDP_TARGET</ffdc>
+    <ffdc>LOC_ENDP_TYPE</ffdc>
+    <ffdc>LOC_ENDP_UNIT_ID</ffdc>
+    <ffdc>LOC_LINK_TRAIN</ffdc>
+    <ffdc>LOC_LINK_TRAIN_NEXT</ffdc>
+    <ffdc>LOC_LINK_FAILED0</ffdc>
+    <ffdc>LOC_LINK_FAILED1</ffdc>
+    <ffdc>REM_ENDP_TARGET</ffdc>
+    <ffdc>REM_ENDP_TYPE</ffdc>
+    <ffdc>REM_ENDP_UNIT_ID</ffdc>
+    <callout>
+      <target>LOC_ENDP_TARGET</target>
+      <priority>MEDIUM</priority>
+    </callout>
+    <callout>
+      <target>REM_ENDP_TARGET</target>
+      <priority>MEDIUM</priority>
+    </callout>
+    <callout>
+      <bus>LOC_ENDP_TARGET,REM_ENDP_TARGET</bus>
+      <priority>HIGH</priority>
+    </callout>
+  </hwpError>
+  <!-- ******************************************************************** -->
+  <hwpError>
     <rc>RC_P9_FAB_IOVALID_TL_NOT_TRAINED_ERR</rc>
     <description>
       Procedure: p9_fab_iovalid