L2 HWPs -- p9_smp_link_layer and p9_fab_iovalid

Tested on n10_e9031_tp030_soa_mc_u008_01
Rebased on e9034 SCOM address header implementation

Change-Id: Iae352ca9392f3e863c9881c90a8a07969befe727
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/22942
Tested-by: Jenkins Server
Tested-by: Auto Mirror
Tested-by: Hostboot CI
Tested-by: PPE CI
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: CHRISTINA L. GRAVES <clgraves@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/24638
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

7 files changed, 427 insertions, 29 deletions

diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.C b/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.C
index 39d0f987d..16c7b3828 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -23,10 +23,10 @@
 ///
 
 //
-// *HWP HW Owner : Michael Dye <dyem@us.ibm.com>
+// *HWP HW Owner : Joe McGill <jmcgill@us.ibm.com>
 // *HWP FW Owner : Thi N. Tran <thi@us.ibm.com>
 // *HWP Team : Nest
-// *HWP Level : 1
+// *HWP Level : 2
 // *HWP Consumed by : HB
 //
 
@@ -34,6 +34,7 @@
 // Includes
 //------------------------------------------------------------------------------
 #include "p9_chiplet_scominit.H"
+#include "p9_fbc_scom.H"
 
 //------------------------------------------------------------------------------
 // Function definitions
@@ -41,11 +42,12 @@
 
 fapi2::ReturnCode p9_chiplet_scominit(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target)
 {
-    FAPI_DBG("Entering ...");
+    fapi2::ReturnCode rc;
+    FAPI_DBG("Start");
 
-    FAPI_DBG("Exiting ...");
-
-//fapi_try_exit:
-    return fapi2::current_err;
+    FAPI_DBG("Invoking p9.fbc.scom.initfile...");
+    FAPI_EXEC_HWP(rc, p9_fbc_scom, i_target);
 
+    FAPI_DBG("End");
+    return rc;
 }
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.mk b/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.mk
index 21de30e4d..cb4257eb7 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.mk
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_chiplet_scominit.mk
@@ -7,7 +7,7 @@
 #
 # EKB Project
 #
-# COPYRIGHT 2015
+# COPYRIGHT 2015,2016
 # [+] International Business Machines Corp.
 #
 #
@@ -17,4 +17,5 @@
 #
 # IBM_PROLOG_END_TAG
 PROCEDURE=p9_chiplet_scominit
+$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9/procedures/hwp/initfiles)
 $(call BUILD_PROCEDURE)
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
index 35d75bdf2..e99fe06bf 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -21,14 +21,13 @@
 /// @brief Manage fabric link iovalid controls (FAPI2)
 ///
 /// @author Joe McGill <jmcgill@us.ibm.com>
-/// @author Christy Graves <clgraves@us.ibm.com>
 ///
 
 //
 // *HWP HWP Owner: Joe McGill <jmcgill@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
-// *HWP Level: 1
+// *HWP Level: 2
 // *HWP Consumed by: HB,FSP
 //
 
@@ -36,18 +35,162 @@
 // Includes
 //------------------------------------------------------------------------------
 #include <p9_fab_iovalid.H>
+#include <p9_fbc_utils.H>
+
+
+//------------------------------------------------------------------------------
+// Constant definitions
+//------------------------------------------------------------------------------
+const uint8_t IOVALID_FIELD_NUM_BITS = 2;
 
 
 //------------------------------------------------------------------------------
 // Function definitions
 //------------------------------------------------------------------------------
 
+///
+/// @brief Manipulate iovalid/FIR settings for a single fabric link
+///
+/// @param[in] i_target Reference to processor chip target
+/// @param[in] i_ctl Reference to link control structure
+/// @param[op] i_set_not_clear Define iovalid operation (true=set, false=clear)
+/// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+///
+fapi2::ReturnCode
+p9_fab_iovalid_update_link(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+                           const p9_fbc_link_ctl_t& i_ctl,
+                           const bool i_set_not_clear)
+{
+    FAPI_DBG("Start");
+
+    // form data buffers for iovalid/RAS FIR mask updates
+    fapi2::buffer<uint64_t> l_iovalid_mask;
+    fapi2::buffer<uint64_t> l_ras_fir_mask;
+
+    if (i_set_not_clear)
+    {
+        // set iovalid
+        l_iovalid_mask.flush<0>();
+        FAPI_TRY(l_iovalid_mask.setBit(i_ctl.iovalid_field_start_bit,
+                                       IOVALID_FIELD_NUM_BITS));
+        // clear RAS FIR mask
+        l_ras_fir_mask.flush<1>();
+        FAPI_TRY(l_ras_fir_mask.clearBit(i_ctl.ras_fir_field_bit));
+    }
+    else
+    {
+        // clear iovalid
+        l_iovalid_mask.flush<1>();
+        FAPI_TRY(l_iovalid_mask.clearBit(i_ctl.iovalid_field_start_bit,
+                                         IOVALID_FIELD_NUM_BITS));
+        // set RAS FIR mask
+        l_ras_fir_mask.flush<0>();
+        FAPI_TRY(l_ras_fir_mask.setBit(i_ctl.ras_fir_field_bit));
+    }
+
+    // use AND/OR mask registers to atomically update link specific fields
+    // in iovalid/RAS FIR mask registers
+    FAPI_TRY(fapi2::putScom(i_target,
+                            (i_set_not_clear) ? (i_ctl.iovalid_or_addr) : (i_ctl.iovalid_clear_addr),
+                            l_iovalid_mask),
+             "Error writing iovalid control register!");
+
+    FAPI_TRY(fapi2::putScom(i_target,
+                            (i_set_not_clear) ? (PU_PB_CENT_SM1_EXTFIR_MASK_REG_AND) : (PU_PB_CENT_SM1_EXTFIR_MASK_REG_OR),
+                            l_ras_fir_mask),
+             "Error writing RAS FIR mask register!");
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
 
 fapi2::ReturnCode
 p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                const bool i_set_not_clear)
 {
     FAPI_INF("Start");
+
+    uint8_t l_x_en_attr[P9_FBC_UTILS_MAX_X_LINKS];
+    uint8_t l_a_en_attr[P9_FBC_UTILS_MAX_A_LINKS];
+    std::vector<std::pair<p9_fbc_link_t, uint8_t>> l_valid_links;
+    std::vector<p9_fbc_link_ctl_t> l_link_ctls(P9_FBC_LINK_CTL_ARR,
+            P9_FBC_LINK_CTL_ARR + (sizeof(P9_FBC_LINK_CTL_ARR) / sizeof(P9_FBC_LINK_CTL_ARR[0])));
+    bool l_ctl_match_found = false;
+
+
+    // read X/A link enable attributes, extract set of valid links
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG,
+                           i_target,
+                           l_x_en_attr),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG");
+
+    for (uint8_t x = 0; x < P9_FBC_UTILS_MAX_X_LINKS; x++)
+    {
+        if (l_x_en_attr[x])
+        {
+            FAPI_DBG("Adding link X%d", x);
+            l_valid_links.push_back(std::make_pair(XBUS, x));
+        }
+        else
+        {
+            FAPI_DBG("Skipping link X%d", x);
+        }
+    }
+
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG,
+                           i_target,
+                           l_a_en_attr),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG");
+
+    for (uint8_t a = 0; a < P9_FBC_UTILS_MAX_A_LINKS; a++)
+    {
+        if (l_a_en_attr[a])
+        {
+            FAPI_DBG("Adding link A%d", a);
+            l_valid_links.push_back(std::make_pair(ABUS, a));
+        }
+        else
+        {
+            FAPI_DBG("Skipping link A%d", a);
+        }
+    }
+
+    // for each valid link, search vector table & call link update routine
+    for (auto l_link_iter = l_valid_links.begin(); l_link_iter != l_valid_links.end(); l_link_iter++)
+    {
+        FAPI_DBG("Processing %s%d (action = %s)",
+                 (l_link_iter->first == XBUS) ? ("X") : ("A)"),
+                 l_link_iter->second,
+                 (i_set_not_clear) ? ("set") : ("clear"));
+
+        l_ctl_match_found = false;
+
+        for (auto l_link_ctl_iter = l_link_ctls.begin();
+             (l_link_ctl_iter != l_link_ctls.end()) && (!l_ctl_match_found);
+             l_link_ctl_iter++)
+        {
+            if ((l_link_ctl_iter->link_type == l_link_iter->first) &&
+                (l_link_ctl_iter->link_id == l_link_iter->second))
+            {
+                l_ctl_match_found = true;
+                FAPI_TRY(p9_fab_iovalid_update_link(i_target,
+                                                    *l_link_ctl_iter,
+                                                    i_set_not_clear),
+                         "Error from p9_fab_iovalid_update_link");
+            }
+        }
+
+        FAPI_ASSERT(l_ctl_match_found,
+                    fapi2::P9_FAB_IOVALID_TABLE_ERR().set_TARGET(i_target).
+                    set_LINK(l_link_iter->first).
+                    set_LINK_ID(l_link_iter->second),
+                    "No match found for link");
+    }
+
+fapi_try_exit:
     FAPI_INF("End");
     return fapi2::current_err;
 }
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
index 8841b5745..a80b3073a 100755
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -32,14 +32,13 @@
 /// the fabric quiesced)
 ///
 /// @author Joe McGill <jmcgill@us.ibm.com>
-/// @author Christy Graves <clgraves@us.ibm.com>
 ///
 
 //
 // *HWP HWP Owner: Joe McGill <jmcgill@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
-// *HWP Level: 1
+// *HWP Level: 2
 // *HWP Consumed by: HB,FSP
 //
 
@@ -71,7 +70,7 @@ extern "C"
 /// @brief Raise/lower iovalids for all logically enabled links on a single chip
 ///
 /// @param[in] i_target Reference to processor chip target
-/// @param[op] i_set_not_clear Define iovalid operation (true=set, false=clear)
+/// @param[in] i_set_not_clear Define iovalid operation (true=set, false=clear)
 /// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
 ///
     fapi2::ReturnCode p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_utils.H b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_utils.H
index 7d472aa1e..7425d97f2 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_fbc_utils.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_fbc_utils.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -39,19 +39,154 @@
 // Includes
 //------------------------------------------------------------------------------
 #include <fapi2.H>
+#include <p9_perv_scom_addresses.H>
+#include <p9_perv_scom_addresses_fld.H>
+#include <p9_misc_scom_addresses.H>
+#include <p9_misc_scom_addresses_fld.H>
+#include <p9_xbus_scom_addresses.H>
+#include <p9_xbus_scom_addresses_fld.H>
+#include <p9_obus_scom_addresses.H>
+
 
 //------------------------------------------------------------------------------
 // Constant definitions
 //------------------------------------------------------------------------------
 
+// link types
+enum p9_fbc_link_t
+{
+    XBUS = 0,
+    ABUS = 1
+};
+
+// link constants
+const uint32_t P9_FBC_UTILS_MAX_X_LINKS = 7;
+const uint32_t P9_FBC_UTILS_MAX_A_LINKS = 4;
+
+// link control structure (DLL/TL/iovalid/FIRs)
+struct p9_fbc_link_ctl_t
+{
+    p9_fbc_link_t link_type;
+    uint8_t       link_id;
+    uint64_t      iovalid_or_addr;
+    uint64_t      iovalid_clear_addr;
+    uint8_t       iovalid_field_start_bit;
+    uint8_t       ras_fir_field_bit;
+    uint64_t      dll_control_addr;
+};
+
+const p9_fbc_link_ctl_t P9_FBC_LINK_CTL_ARR[P9_FBC_UTILS_MAX_X_LINKS + P9_FBC_UTILS_MAX_A_LINKS] =
+{
+    {
+        XBUS,
+        0,
+        PERV_XB_CPLT_CONF1_OR,
+        PERV_XB_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X0_FIR_ERR,
+        XBUS_0_LL0_IOEL_CONTROL
+    },
+    {
+        XBUS,
+        1,
+        PERV_XB_CPLT_CONF1_OR,
+        PERV_XB_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_6D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X1_FIR_ERR,
+        XBUS_1_LL1_IOEL_CONTROL
+    },
+    {
+        XBUS,
+        2,
+        PERV_XB_CPLT_CONF1_OR,
+        PERV_XB_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_8D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X2_FIR_ERR,
+        XBUS_2_LL2_IOEL_CONTROL
+    },
+    {
+        XBUS,
+        3,
+        PERV_OB0_CPLT_CONF1_OR,
+        PERV_OB0_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X3_FIR_ERR,
+        OBUS_0_LL0_IOOL_CONTROL
+    },
+    {
+        XBUS,
+        4,
+        PERV_OB1_CPLT_CONF1_OR,
+        PERV_OB1_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X4_FIR_ERR,
+        OBUS_1_LL3_IOOL_CONTROL
+    },
+    {
+        XBUS,
+        5,
+        PERV_OB2_CPLT_CONF1_OR,
+        PERV_OB2_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X5_FIR_ERR,
+        OBUS_2_LL3_IOOL_CONTROL
+    },
+    {
+        XBUS,
+        6,
+        PERV_OB3_CPLT_CONF1_OR,
+        PERV_OB3_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X6_FIR_ERR,
+        OBUS_3_LL3_IOOL_CONTROL
+    },
+    {
+        ABUS,
+        0,
+        PERV_OB0_CPLT_CONF1_OR,
+        PERV_OB0_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X3_FIR_ERR,
+        OBUS_0_LL0_IOOL_CONTROL
+    },
+    {
+        ABUS,
+        1,
+        PERV_OB1_CPLT_CONF1_OR,
+        PERV_OB1_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X4_FIR_ERR,
+        OBUS_1_LL3_IOOL_CONTROL
+    },
+    {
+        ABUS,
+        2,
+        PERV_OB2_CPLT_CONF1_OR,
+        PERV_OB2_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X5_FIR_ERR,
+        OBUS_2_LL3_IOOL_CONTROL
+    },
+    {
+        ABUS,
+        3,
+        PERV_OB3_CPLT_CONF1_OR,
+        PERV_OB3_CPLT_CONF1_CLEAR,
+        PERV_1_CPLT_CONF1_IOVALID_4D,
+        PU_PB_CENT_SM1_EXTFIR_MASK_REG_PB_X6_FIR_ERR,
+        OBUS_3_LL3_IOOL_CONTROL
+    }
+};
+
 // address range definitions
 const uint64_t P9_FBC_UTILS_FBC_MAX_ADDRESS = ((1ULL << 56) - 1ULL);
 const uint64_t P9_FBC_UTILS_CACHELINE_MASK = 0x7FULL;
 const uint64_t P9_FBC_UTILS_LAST_ADDR_IN_CACHELINE = 0x78ULL;
 
-// cacheline size = 128
+// cacheline size = 128B
 const uint64_t FABRIC_CACHELINE_SIZE = 0x80;
 
+
 //------------------------------------------------------------------------------
 // Function prototypes
 //------------------------------------------------------------------------------
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.C b/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.C
index 75b5b5a9e..68de9b3b8 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -18,17 +18,16 @@
 /* IBM_PROLOG_END_TAG                                                     */
 ///
 /// @file p9_smp_link_layer.C
-/// @brief Enable fabric link/transaction layers (FAPI2)
+/// @brief Start SMP link layer (FAPI2)
 ///
 /// @author Joe McGill <jmcgill@us.ibm.com>
-/// @author Christy Graves <clgraves@us.ibm.com>
 ///
 
 //
 // *HWP HWP Owner: Joe McGill <jmcgill@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
-// *HWP Level: 1
+// *HWP Level: 2
 // *HWP Consumed by: HB,FSP
 //
 
@@ -36,17 +35,132 @@
 // Includes
 //------------------------------------------------------------------------------
 #include <p9_smp_link_layer.H>
+#include <p9_fbc_utils.H>
+
+
+//------------------------------------------------------------------------------
+// Constant definitions
+//------------------------------------------------------------------------------
+// IOO/IOL control registers share common layout
+const uint32_t DLL_CONTROL_LINK0_STARTUP_BIT = XBUS_LL0_IOEL_CONTROL_LINK0_STARTUP;
+const uint32_t DLL_CONTROL_LINK1_STARTUP_BIT = XBUS_LL0_IOEL_CONTROL_LINK1_STARTUP;
 
 
 //------------------------------------------------------------------------------
 // Function definitions
 //------------------------------------------------------------------------------
 
+///
+/// @brief Engage DLL/TL training for a single fabric link
+///
+/// @param[in] i_target Reference to processor chip target
+/// @param[in] i_ctl Reference to link control structure
+/// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
+///
+fapi2::ReturnCode
+p9_smp_link_layer_train_link(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+                             const p9_fbc_link_ctl_t& i_ctl)
+
+{
+    FAPI_DBG("Start");
+
+    fapi2::buffer<uint64_t> l_dll_control;
+
+    FAPI_TRY(fapi2::getScom(i_target, i_ctl.dll_control_addr, l_dll_control),
+             "Error reading DLL control register!");
+    l_dll_control.setBit<DLL_CONTROL_LINK0_STARTUP_BIT>();
+    l_dll_control.setBit<DLL_CONTROL_LINK1_STARTUP_BIT>();
+    FAPI_TRY(fapi2::putScom(i_target, i_ctl.dll_control_addr, l_dll_control),
+             "Error writing DLL control register!");
+
+fapi_try_exit:
+    FAPI_DBG("End");
+    return fapi2::current_err;
+}
+
+
 
 fapi2::ReturnCode
 p9_smp_link_layer(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target)
 {
     FAPI_INF("Start");
+
+    uint8_t l_x_en_attr[P9_FBC_UTILS_MAX_X_LINKS];
+    uint8_t l_a_en_attr[P9_FBC_UTILS_MAX_A_LINKS];
+    std::vector<std::pair<p9_fbc_link_t, uint8_t>> l_valid_links;
+    std::vector<p9_fbc_link_ctl_t> l_link_ctls(P9_FBC_LINK_CTL_ARR,
+            P9_FBC_LINK_CTL_ARR + (sizeof(P9_FBC_LINK_CTL_ARR) / sizeof(P9_FBC_LINK_CTL_ARR[0])));
+    bool l_ctl_match_found = false;
+
+
+    // read X/A link enable attributes, extract set of valid links
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG,
+                           i_target,
+                           l_x_en_attr),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG");
+
+    for (uint8_t x = 0; x < P9_FBC_UTILS_MAX_X_LINKS; x++)
+    {
+        if (l_x_en_attr[x])
+        {
+            FAPI_DBG("Adding link X%d", x);
+            l_valid_links.push_back(std::make_pair(XBUS, x));
+        }
+        else
+        {
+            FAPI_DBG("Skipping link X%d", x);
+        }
+    }
+
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG,
+                           i_target,
+                           l_a_en_attr),
+             "Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG");
+
+    for (uint8_t a = 0; a < P9_FBC_UTILS_MAX_A_LINKS; a++)
+    {
+        if (l_a_en_attr[a])
+        {
+            FAPI_DBG("Adding link A%d", a);
+            l_valid_links.push_back(std::make_pair(ABUS, a));
+        }
+        else
+        {
+            FAPI_DBG("Skipping link A%d", a);
+        }
+    }
+
+    // for each valid link, search vector table & call link update routine
+    for (auto l_link_iter = l_valid_links.begin(); l_link_iter != l_valid_links.end(); l_link_iter++)
+    {
+        FAPI_DBG("Processing %s%d",
+                 (l_link_iter->first == XBUS) ? ("X") : ("A)"),
+                 l_link_iter->second);
+
+        l_ctl_match_found = false;
+
+        for (auto l_link_ctl_iter = l_link_ctls.begin();
+             (l_link_ctl_iter != l_link_ctls.end()) && (!l_ctl_match_found);
+             l_link_ctl_iter++)
+        {
+            if ((l_link_ctl_iter->link_type == l_link_iter->first) &&
+                (l_link_ctl_iter->link_id == l_link_iter->second))
+            {
+                l_ctl_match_found = true;
+                FAPI_TRY(p9_smp_link_layer_train_link(i_target,
+                                                      *l_link_ctl_iter),
+                         "Error from p9_smp_link_layer_train_link");
+            }
+        }
+
+        FAPI_ASSERT(l_ctl_match_found,
+                    fapi2::P9_SMP_LINK_LAYER_TABLE_ERR().set_TARGET(i_target).
+                    set_LINK(l_link_iter->first).
+                    set_LINK_ID(l_link_iter->second),
+                    "No match found for link");
+    }
+
+fapi_try_exit:
     FAPI_INF("End");
     return fapi2::current_err;
 }
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.H b/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.H
index 462541419..015a831d1 100755
--- a/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_smp_link_layer.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -18,21 +18,25 @@
 /* IBM_PROLOG_END_TAG                                                     */
 ///
 /// @file p9_smp_link_layer.H
-/// @brief Enable fabric link/transaction layers (FAPI2)
+/// @brief Start SMP link layer (FAPI2)
 ///
-/// Enable fabric X/A data link (DL) and transaction (TL) layers
+/// Train fabric Data Link Layer (DLL) and Transaction Layer (TL).
 ///
-/// Procedure should enable use of link TL mailbox
+/// High level sequence:
+/// - HWP engages DLL training via SCOM
+/// - HW fires DLL link up FIR bit when finished
+/// - Link up FIR bit launches TL training, in HW
+/// - HW fires TL training done FIR bit when finished.
+///   mailbox registers should be accessible for HWP/FW use
 ///
 /// @author Joe McGill <jmcgill@us.ibm.com>
-/// @author Christy Graves <clgraves@us.ibm.com>
 ///
 
 //
 // *HWP HWP Owner: Joe McGill <jmcgill@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
-// *HWP Level: 1
+// *HWP Level: 2
 // *HWP Consumed by: HB,FSP
 //
 
@@ -60,7 +64,7 @@ extern "C"
 {
 
 ///
-/// @brief Enable fabric data link/transaction layers, for all logically enabled links on a single chip
+/// @brief Train fabric DLL/TL layers
 ///
 /// @param[in] i_target Reference to processor chip target
 /// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.