istep9 HWP enablement

Change-Id: Ie780ed10e29961092a19373c994ec24a8cea500a
RTC:146576
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/23517
Tested-by: Jenkins Server
Tested-by: FSP CI Jenkins
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

9 files changed, 762 insertions, 459 deletions

diff --git a/src/include/usr/fapi2/plat_trace.H b/src/include/usr/fapi2/plat_trace.H
index 9d03adcff..21915753e 100644
--- a/src/include/usr/fapi2/plat_trace.H
+++ b/src/include/usr/fapi2/plat_trace.H
@@ -38,6 +38,8 @@
 // Use the same macro from FAPI1
 #include <fapiPlatTrace.H>
 
-static const uint32_t MAX_ECMD_STRING_LEN = 64;
-
+namespace fapi2
+{
+  static const uint32_t MAX_ECMD_STRING_LEN = 64;
+}
 #endif // PLAT_TRACE_H_
diff --git a/src/usr/isteps/istep09/call_fabric_erepair.C b/src/usr/isteps/istep09/call_fabric_erepair.C
index 005491860..44422b485 100644
--- a/src/usr/isteps/istep09/call_fabric_erepair.C
+++ b/src/usr/isteps/istep09/call_fabric_erepair.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,14 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include    <pbusLinkSvc.H>
+
+#include <fapi2/target.H>
+#include <fapi2/plat_hwp_invoker.H>
+#include <errl/errlmanager.H>
+
+// HWP procedure
+#include <p9_io_xbus_restore_erepair.H>
 
 namespace   ISTEP_09
 {
@@ -68,14 +76,37 @@ using   namespace   TARGETING;
 using   namespace   HWAS;
 
 
+// prototype
+/**
+ *  @brief Try to restore target's endpoints
+ *
+ *  Runs restore_erepair HWP on each group combination for this target
+ *  This hardware procedure should update the bad lane vector and
+ *  power down the bad lanes.
+ *
+ *  @param[in]  i_target       XBUS target endpoint
+ *  @param[in]  i_rx_bad_lanes Vector of Rx Bad Lanes
+ *  @param[in]  i_tx_bad_lanes Vector of Tx Bad Lanes
+ *  @param[out] o_step_error   Failing error logs added to this
+ *
+ *  @pre Target service must be initialized
+ *  @post See "return"
+ *  @return uint8_t Number of failures
+ *  @retval 0 means successfully restored target endpoint
+ *  @retval 4 Means all attempts to restore/erepair failed
+ */
+uint8_t restore_endpoint(const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_target,
+                         const std::vector< uint8_t >& i_rx_bad_lanes,
+                         const std::vector< uint8_t >& i_tx_bad_lanes,
+                         ISTEP_ERROR::IStepError& o_step_error);
+
 //
 //  Wrapper function to call fabric_erepair
 //
 void*    call_fabric_erepair( void    *io_pArgs )
 {
     errlHndl_t l_errl = NULL;
-    //@TODO RTC:134079
-    /*ISTEP_ERROR::IStepError l_StepError;
+    ISTEP_ERROR::IStepError l_StepError;
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_fabric_erepair entry" );
 
     do {
@@ -93,8 +124,9 @@ void*    call_fabric_erepair( void    *io_pArgs )
     {
         // Call to check Processor SBE SEEPROM Images against NEST_FREQ_MHZ
         // attributes and make any necessary updates
-        l_errl = SBE::updateProcessorSbeSeeproms(
-                            SBE::SBE_UPDATE_ONLY_CHECK_NEST_FREQ);
+        // TODO-RTC:138226 - add it after SBE is ported to fapi2
+        //l_errl = SBE::updateProcessorSbeSeeproms(
+        //                SBE::SBE_UPDATE_ONLY_CHECK_NEST_FREQ);
 
         if (l_errl)
         {
@@ -104,24 +136,28 @@ void*    call_fabric_erepair( void    *io_pArgs )
             errlCommit( l_errl, HWPF_COMP_ID );
             break;
         }
-
     }
 
     std::vector<uint8_t> l_endp1_txFaillanes;
     std::vector<uint8_t> l_endp1_rxFaillanes;
     std::vector<uint8_t> l_endp2_txFaillanes;
     std::vector<uint8_t> l_endp2_rxFaillanes;
+    uint8_t l_restore_failures = 0;
+
+    EDI_EI_INITIALIZATION::TargetPairs_t l_PbusConnections;
+    const uint32_t MaxBusSet = 1;
+    TYPE busSet[MaxBusSet] = { TYPE_XBUS }; // TODO RTC:152304 - add TYPE_OBUS
 
-    TargetPairs_t l_PbusConnections;
-    const uint32_t MaxBusSet = 2;
-    TYPE busSet[MaxBusSet] = { TYPE_ABUS, TYPE_XBUS };
     uint32_t l_count = 0;
-    fapi::TargetType l_tgtType = fapi::TARGET_TYPE_NONE;
+    fapi2::TargetType l_tgtType = fapi2::TARGET_TYPE_NONE;
+
+    TARGETING::ATTR_FAPI_NAME_type  l_target_name = {0};
 
     for (uint32_t i = 0; l_StepError.isNull() && (i < MaxBusSet); i++)
     {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                            l_PbusConnections, busSet[i] );
+        // grab the unique pairs for this particular type
+        l_errl = EDI_EI_INITIALIZATION::PbusLinkSvc::getTheInstance().
+                    getPbusConnections( l_PbusConnections, busSet[i] );
         if ( l_errl )
         {
             // Create IStep error log and cross reference error that occurred
@@ -129,27 +165,32 @@ void*    call_fabric_erepair( void    *io_pArgs )
 
             // Commit Error
             errlCommit( l_errl, HWPF_COMP_ID );
+
+            break;
         }
 
-        for (TargetPairs_t::const_iterator l_itr = l_PbusConnections.begin();
-             (l_StepError.isNull()) && (l_itr != l_PbusConnections.end());
-             ++l_itr)
+        for (const auto & l_PbusConnection: l_PbusConnections)
         {
-            const fapi::Target l_fapi_endp1_target(
-                   (i ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->first)));
-            const fapi::Target l_fapi_endp2_target(
-                   (i ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->second)));
+            const TARGETING::Target* l_thisPbusTarget = l_PbusConnection.first;
+            const TARGETING::Target* l_connectedPbusTarget =
+                                                    l_PbusConnection.second;
+
+            // TODO-RTC:152304 - need to adjust target types if adding OBUS/ABUS
+            const fapi2::Target<fapi2::TARGET_TYPE_XBUS> l_fapi_endp1_target
+              (const_cast<TARGETING::Target*>(l_thisPbusTarget));
+
+            const fapi2::Target<fapi2::TARGET_TYPE_XBUS> l_fapi_endp2_target
+              (const_cast<TARGETING::Target*>(l_connectedPbusTarget));
 
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "===== " );
 
             // Get the repair lanes from the VPD
-            fapi::ReturnCode l_rc;
+            fapi2::ReturnCode l_rc;
             l_endp1_txFaillanes.clear();
             l_endp1_rxFaillanes.clear();
             l_endp2_txFaillanes.clear();
             l_endp2_rxFaillanes.clear();
+/* TODO-RTC:134079 - L2 HWP enablement
             l_rc = erepairGetRestoreLanes(l_fapi_endp1_target,
                                           l_endp1_txFaillanes,
                                           l_endp1_rxFaillanes,
@@ -166,8 +207,8 @@ void*    call_fabric_erepair( void    *io_pArgs )
                 l_errl = fapiRcToErrl(l_rc);
 
                 // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
+                ErrlUserDetailsTarget(l_thisPbusTarget).addToLog( l_errl );
+                ErrlUserDetailsTarget(l_connectedPbusTarget).addToLog( l_errl );
 
                 // Create IStep error log and cross ref error that occurred
                 l_StepError.addErrorDetails( l_errl);
@@ -176,109 +217,97 @@ void*    call_fabric_erepair( void    *io_pArgs )
                 errlCommit(l_errl, HWPF_COMP_ID);
                 break;
             }
-
+*/
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
                    "===== Call io_restore_erepair HWP"
-                   "%cbus connection ", (i ? 'X' : 'A') );
+                   "%cbus connection ", (i ? 'X' : 'O') );
 
             if(l_endp1_txFaillanes.size() || l_endp1_rxFaillanes.size())
             {
-                // call the io_restore_erepair HWP to restore eRepair
+                // call the io_xbus_restore_erepair HWP to restore eRepair
                 // lanes of endp1
-                //@TODO RTC:133830
-                //FAPI_INVOKE_HWP(l_errl,
-                //                p9_io_restore_erepair,
-                //                l_fapi_endp1_target,
-                //                l_endp1_txFaillanes,
-                //                l_endp1_rxFaillanes);
+                l_restore_failures = restore_endpoint(l_fapi_endp1_target,
+                                                      l_endp1_rxFaillanes,
+                                                      l_endp1_txFaillanes,
+                                                      l_StepError);
             }
 
-            if(l_errl)
-            {
-                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                        "ERROR 0x%.8X :  io_restore_erepair HWP"
-                        "%cbus connection ",
-                        l_errl->reasonCode(), (i ? 'X' : 'A') );
-
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
-
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl);
+            fapi2::toString(l_fapi_endp1_target,
+                            l_target_name,
+                            sizeof(l_target_name));
 
-                // Commit Error
-                errlCommit(l_errl, HWPF_COMP_ID);
-                break;
-            }
-
-            l_tgtType = l_fapi_endp1_target.getType();
-            for(l_count = 0; l_count < l_endp1_txFaillanes.size(); l_count++)
+            if (l_restore_failures)
             {
-                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,"Successfully"
-                          " restored Tx lane %d, of %s, of endpoint %s",
-                          l_endp1_txFaillanes[l_count],
-                          l_tgtType == TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
-                          "A-Bus", l_fapi_endp1_target.toEcmdString());
+                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%d restore endpoint attempts to endpoint1 %s failed.",
+                    l_restore_failures, l_target_name);
             }
-
-            for(l_count = 0; l_count < l_endp1_rxFaillanes.size(); l_count++)
+            else
             {
-                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,"Successfully"
-                          " restored Rx lane %d, of %s, of endpoint %s",
-                          l_endp1_rxFaillanes[l_count],
-                          l_tgtType == TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
-                          "A-Bus", l_fapi_endp1_target.toEcmdString());
+                l_tgtType = l_fapi_endp1_target.getType();
+
+                for(l_count = 0; l_count < l_endp1_txFaillanes.size();l_count++)
+                {
+                    fapi2::toString(l_fapi_endp1_target,
+                                    l_target_name,
+                                    sizeof(l_target_name));
+                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,"Successfully"
+                        " restored Tx lane %d, of %s, of endpoint %s",
+                        l_endp1_txFaillanes[l_count],
+                        l_tgtType == fapi2::TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus":
+                        "O-Bus", l_target_name);
+                }
+
+                for(l_count = 0; l_count < l_endp1_rxFaillanes.size();l_count++)
+                {
+                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace, "Successfully"
+                        " restored Rx lane %d, of %s, of endpoint %s",
+                        l_endp1_rxFaillanes[l_count],
+                        l_tgtType == fapi2::TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus":
+                        "O-Bus", l_target_name);
+                }
             }
 
             if(l_endp2_txFaillanes.size() || l_endp2_rxFaillanes.size())
             {
-                // call the io_restore_erepair HWP to restore eRepair
+                // call the io_xbus_restore_erepair HWP to restore eRepair
                 // lanes of endp2
-                //@TODO RTC:133830
-                FAPI_INVOKE_HWP(l_errl,
-                                io_restore_erepair,
-                                l_fapi_endp2_target,
-                                l_endp2_txFaillanes,
-                                l_endp2_rxFaillanes);
+                l_restore_failures = restore_endpoint(l_fapi_endp2_target,
+                                                      l_endp2_rxFaillanes,
+                                                      l_endp2_txFaillanes,
+                                                      l_StepError);
             }
 
-            if (l_errl)
+            fapi2::toString(l_fapi_endp2_target,
+                                l_target_name,
+                                sizeof(l_target_name));
+
+            if ( l_restore_failures )
             {
                 TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                            "ERROR 0x%.8X :  io_restore_erepair HWP"
-                            "%cbus connection ",
-                            l_errl->reasonCode(), (i ? 'X' : 'A') );
-
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
-
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl);
+                    "%d restore endpoint attempts to endpoint2 %s failed.",
+                    l_restore_failures, l_target_name);
 
-                // Commit Error
-                errlCommit(l_errl, HWPF_COMP_ID);
-                break;
+                continue;
             }
 
             l_tgtType = l_fapi_endp2_target.getType();
             for(l_count = 0; l_count < l_endp2_txFaillanes.size(); l_count++)
             {
                 TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,"Successfully"
-                          " restored Tx lane %d, of %s, of endpoint %s",
-                          l_endp2_txFaillanes[l_count],
-                          l_tgtType == TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
-                          "A-Bus", l_fapi_endp2_target.toEcmdString());
+                      " restored Tx lane %d, of %s, of endpoint %s",
+                      l_endp2_txFaillanes[l_count],
+                      l_tgtType == fapi2::TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
+                      "O-Bus", l_target_name);
             }
 
             for(l_count = 0; l_count < l_endp2_rxFaillanes.size(); l_count++)
             {
                 TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,"Successfully"
-                          " restored Rx lane %d, of %s, of endpoint %s",
-                          l_endp2_rxFaillanes[l_count],
-                          l_tgtType == TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
-                          "A-Bus", l_fapi_endp2_target.toEcmdString());
+                      " restored Rx lane %d, of %s, of endpoint %s",
+                      l_endp2_rxFaillanes[l_count],
+                      l_tgtType == fapi2::TARGET_TYPE_XBUS_ENDPOINT ? "X-Bus" :
+                      "O-Bus", l_target_name);
             }
         } // end for l_PbusConnections
     } // end for MaxBusSet
@@ -288,7 +317,71 @@ void*    call_fabric_erepair( void    *io_pArgs )
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_fabric_erepair exit" );
 
     return l_StepError.getErrorHandle();
-    */
-        return l_errl;
 }
+
+
+uint8_t restore_endpoint(const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_target,
+                         const std::vector< uint8_t >& i_rx_bad_lanes,
+                         const std::vector< uint8_t >& i_tx_bad_lanes,
+                         ISTEP_ERROR::IStepError & o_step_error)
+{
+    uint8_t o_failures = 0;
+    errlHndl_t l_errl = NULL;
+
+    fapi2::TargetType l_tgtType = fapi2::TARGET_TYPE_NONE;
+    l_tgtType = i_target.getType();
+
+    // clock group is either 0 or 1
+    // need to train both groups and allow for them to differ
+    uint8_t l_this_group = 0;
+    uint8_t l_connected_group = 0;
+    uint8_t l_group_loop = 0;
+    for (l_group_loop = 0; l_group_loop < 4; l_group_loop++)
+    {
+        l_this_group = l_group_loop / 2;      // 0, 0, 1, 1
+        l_connected_group = l_group_loop % 2; // 0, 1, 1, 0
+
+
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                 "Running p9_io_xbus_restore_erepair HWP on "
+                 "this %C-BUS target %.8X (groups %d and %d)",
+                 (l_tgtType == fapi2::TARGET_TYPE_XBUS_ENDPOINT) ? 'X':'O',
+                 TARGETING::get_huid(i_target),
+                 l_this_group, l_connected_group );
+
+        /*
+         * A HWP that runs Restore eRepair.
+         * This procedure should update the
+         * bad lane vector and power down the bad lanes.
+         */
+        FAPI_INVOKE_HWP(l_errl,
+                        p9_io_xbus_restore_erepair,
+                        i_target,
+                        l_this_group,
+                        i_rx_bad_lanes,
+                        i_target,
+                        l_connected_group,
+                        i_tx_bad_lanes);
+
+        if (l_errl)
+        {
+            o_failures++;
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+              "ERROR 0x%.8X :  io_xbus_restore_erepair HWP "
+              "xbus connection.", l_errl->reasonCode() );
+
+            // capture the target data in the elog
+            ErrlUserDetailsTarget(i_target).addToLog( l_errl );
+
+            // Create IStep error log and cross ref error that occurred
+            o_step_error.addErrorDetails( l_errl);
+
+            // Commit Error
+            errlCommit(l_errl, HWPF_COMP_ID);
+            l_errl = NULL;
+        }
+    }
+    return o_failures;
+}
+
 };
diff --git a/src/usr/isteps/istep09/call_fabric_io_dccal.C b/src/usr/isteps/istep09/call_fabric_io_dccal.C
index 0b49bed69..0dbfd9d48 100644
--- a/src/usr/isteps/istep09/call_fabric_io_dccal.C
+++ b/src/usr/isteps/istep09/call_fabric_io_dccal.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,13 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include  <pbusLinkSvc.H>
+#include  <fapi2/target.H>
+#include  <fapi2/plat_hwp_invoker.H>
+
+// HWP
+#include    <p9_io_xbus_dccal.H>
+
 namespace   ISTEP_09
 {
 using   namespace   ISTEP;
@@ -69,9 +76,8 @@ using   namespace   HWAS;
 //
 void*    call_fabric_io_dccal( void    *io_pArgs )
 {
-    errlHndl_t  l_errl  =   NULL;
-    //@TODO RTC:134079 Port to fapi2
-/*    IStepError  l_StepError;
+    errlHndl_t  l_errl = NULL;
+    IStepError  l_StepError;
 
     // We are not running this analog procedure in VPO
     if (TARGETING::is_vpo())
@@ -84,83 +90,117 @@ void*    call_fabric_io_dccal( void    *io_pArgs )
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_io_dccal entry" );
 
-    TargetPairs_t l_PbusConnections;
-    TargetPairs_t::iterator l_itr;
-    const uint32_t MaxBusSet = 2;
-    TYPE busSet[MaxBusSet] = { TYPE_ABUS, TYPE_XBUS };
-
-    // Note:
-    // Due to lab tester board environment, HW procedure writer (Varkey) has
-    // requested to send in one target of a time (we used to send in
-    // both ends in one call). Even though they don't have to be
-    // in order, we should keep the pair concept here in case we need to send
-    // in a pair in the future again.
+    EDI_EI_INITIALIZATION::TargetPairs_t l_PbusConnections;
+    const uint32_t MaxBusSet = 1;
+    TYPE busSet[MaxBusSet] = { TYPE_XBUS }; // TODO RTC:152304 - add TYPE_OBUS
+
     for (uint32_t ii = 0; (!l_errl) && (ii < MaxBusSet); ii++)
     {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                            l_PbusConnections, busSet[ii] );
-
-        for (l_itr = l_PbusConnections.begin();
-             l_itr != l_PbusConnections.end();
-             ++l_itr)
+        l_errl = EDI_EI_INITIALIZATION::PbusLinkSvc::getTheInstance().
+                    getPbusConnections(l_PbusConnections, busSet[ii]);
+        if (l_errl)
         {
-            const fapi::Target l_fapi_endp1_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->first)));
-            const fapi::Target l_fapi_endp2_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->second)));
-
-            //@TODO RTC:133830  call the HWP with each bus connection
-            //FAPI_INVOKE_HWP( l_errl, io_dccal, l_fapi_endp1_target );
-
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "%s : %cbus connection fabric_io_dccal. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->first) );
-            if ( l_errl )
-            {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
+                "ERROR 0x%.8X : getPbusConnections TYPE_%cBUS returns error",
+                l_errl->reasonCode(), (ii ? 'X':'O') );
+
+            // Create IStep error log and cross reference to error that occurred
+            l_StepError.addErrorDetails( l_errl );
 
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
+            // Commit the error log
+            // Log should be deleted and set to NULL in errlCommit.
+            errlCommit(l_errl, HWPF_COMP_ID);
 
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
+            // Don't continue with a potential bad connection set
+            break;
+        }
 
-            //@TODO RTC:133830  call the HWP with each bus connection
-            FAPI_INVOKE_HWP( l_errl, fabric_io_dccal, l_fapi_endp2_target );
+        for (const auto & l_PbusConnection: l_PbusConnections)
+        {
+            const TARGETING::Target* l_thisPbusTarget = l_PbusConnection.first;
+            const TARGETING::Target* l_connectedPbusTarget =
+                                                    l_PbusConnection.second;
 
-            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "%s : %cbus connection fabric_io_dccal. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->second) );
-            if ( l_errl )
-            {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_thisPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_thisPbusTarget)));
 
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_connectedPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_connectedPbusTarget)));
 
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
-        }
-    }
+            // group is either 0 or 1
+            uint8_t l_group = 0;
+            for (l_group = 0; l_group < 2; l_group++)
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                         "Running p9_io_xbus_dccal HWP on "
+                         "this %cbus target %.8X, group %d",
+                         (ii ? 'X' : 'O'),
+                         TARGETING::get_huid(l_thisPbusTarget), l_group );
+
+                FAPI_INVOKE_HWP( l_errl, p9_io_xbus_dccal,
+                                 l_thisPbusFapi2Target, l_group );
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%s : %cbus connection p9_io_xbus_dccal "
+                    "with group %d. Target 0x%.8X",
+                    (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'O'),
+                    l_group, TARGETING::get_huid(l_thisPbusTarget) );
+                if ( l_errl )
+                {
+                    // capture the target data in the elog
+                    ErrlUserDetailsTarget(l_thisPbusTarget).addToLog(l_errl);
+
+                    // Create IStep error log and cross ref error that occurred
+                    l_StepError.addErrorDetails( l_errl );
+
+                    // Commit Error
+                    errlCommit( l_errl, HWPF_COMP_ID );
+                    // We want to continue the training despite the error, so
+                    // no break
+                    l_errl = NULL;
+                }
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "Running p9_io_xbus_dccal HWP on "
+                    "connected %cbus target %.8X, group %d",
+                    (ii ? 'X' : 'O'),
+                    TARGETING::get_huid(l_connectedPbusTarget), l_group );
+
+                // call the HWP with each bus connection
+                FAPI_INVOKE_HWP( l_errl, p9_io_xbus_dccal,
+                                 l_connectedPbusFapi2Target, l_group);
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%s : %cbus connection p9_io_xbus_dccal "
+                    "with group %d. Target 0x%.8X",
+                    (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'O'),
+                    l_group, TARGETING::get_huid(l_connectedPbusTarget) );
+                if ( l_errl )
+                {
+                    // capture the target data in the elog
+                    ErrlUserDetailsTarget(l_connectedPbusTarget).
+                        addToLog(l_errl);
+
+                    // Create IStep error log and cross ref error that occurred
+                    l_StepError.addErrorDetails( l_errl );
+
+                    // Commit Error
+                    errlCommit( l_errl, HWPF_COMP_ID );
+                    // We want to continue the training despite the error, so
+                    // no break
+                    l_errl = NULL;
+                }
+            } // end of groups
+        } // end of connection TYPE combinations
+    } // end of connection set loop
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_io_dccal exit" );
 
     // end task, returning any errorlogs to IStepDisp
     return l_StepError.getErrorHandle();
-    */
-        return l_errl;
 }
+
 };
diff --git a/src/usr/isteps/istep09/call_fabric_io_run_training.C b/src/usr/isteps/istep09/call_fabric_io_run_training.C
index a9a4a78cf..65b424106 100644
--- a/src/usr/isteps/istep09/call_fabric_io_run_training.C
+++ b/src/usr/isteps/istep09/call_fabric_io_run_training.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -53,19 +53,34 @@
 #include    <sbe/sbeif.H>
 
 //  targeting support
+#include   <targeting/common/attributes.H>
+#include   <targeting/common/targetservice.H>
+
 #include    <targeting/common/commontargeting.H>
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include  <pbusLinkSvc.H>
+#include  <fapi2/target.H>
+#include  <fapi2/plat_hwp_invoker.H>
+
+// HWP
+#include    <p9_io_xbus_linktrain.H>
+
 namespace   ISTEP_09
 {
 
-
 using   namespace   ISTEP;
 using   namespace   ISTEP_ERROR;
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   HWAS;
+
+// helper function prototypes
+uint8_t run_linktraining(
+                const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_master_target,
+                const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_slave_target,
+                ISTEP_ERROR::IStepError & o_step_error );
 //
 //  Wrapper function to call fabric_io_run_training
 //
@@ -73,71 +88,182 @@ void*    call_fabric_io_run_training( void    *io_pArgs )
 {
 
     IStepError  l_StepError;
-    //@TODO RTC:134079
-/*    errlHndl_t  l_errl  =   NULL;
+    errlHndl_t  l_errl  =   NULL;
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_io_run_training entry" );
 
-    TargetPairs_t l_PbusConnections;
-    const uint32_t MaxBusSet = 2;
-
-    // Note: Run XBUS first to match with Cronus
-    TYPE busSet[MaxBusSet] = { TYPE_XBUS, TYPE_ABUS };
+    uint8_t l_linktrain_failures = 0;
+    EDI_EI_INITIALIZATION::TargetPairs_t l_PbusConnections;
+    const uint32_t MaxBusSet = 1;
+    TYPE busSet[MaxBusSet] = { TYPE_XBUS }; // TODO RTC:152304 - add TYPE_OBUS
 
-    for (uint32_t i = 0; l_StepError.isNull() && (i < MaxBusSet); i++)
+    for (uint32_t ii = 0; (!l_errl) && (ii < MaxBusSet); ii++)
     {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                            l_PbusConnections, busSet[i] );
-
-        if ( l_errl )
+        l_errl = EDI_EI_INITIALIZATION::PbusLinkSvc::getTheInstance().
+                    getPbusConnections(l_PbusConnections, busSet[ii]);
+        if (l_errl)
         {
-            // Create IStep error log and cross reference error that occurred
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                "ERROR 0x%.8X : getPbusConnections TYPE_%cBUS returns error",
+                l_errl->reasonCode(), (ii ? 'X':'O') );
+
+            // Create IStep error log and cross reference to error that occurred
             l_StepError.addErrorDetails( l_errl );
 
-            // Commit Error
-            errlCommit( l_errl, HWPF_COMP_ID );
+            // Commit the error log
+            // Log should be deleted and set to NULL in errlCommit.
+            errlCommit(l_errl, HWPF_COMP_ID);
+
+            // Don't continue with a potential bad connection set
+            break;
         }
 
-        for (TargetPairs_t::const_iterator l_itr = l_PbusConnections.begin();
-             (l_StepError.isNull()) && (l_itr != l_PbusConnections.end());
-             ++l_itr)
+        for (const auto & l_PbusConnection: l_PbusConnections)
         {
-            const fapi::Target l_fapi_endp1_target(
-                   (i ? TARGET_TYPE_ABUS_ENDPOINT : TARGET_TYPE_XBUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->first)));
-            const fapi::Target l_fapi_endp2_target(
-                   (i ? TARGET_TYPE_ABUS_ENDPOINT : TARGET_TYPE_XBUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->second)));
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_thisPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_PbusConnection.first)));
 
-            //@TODO RTC:133830  call the HWP with each bus connection
-            //FAPI_INVOKE_HWP( l_errl, p9_io_run_training,
-            //                 l_fapi_endp1_target, l_fapi_endp2_target );
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_connectedPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_PbusConnection.second)));
 
-            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "%s : %cbus connection io_run_training",
-                       (l_errl ? "ERROR" : "SUCCESS"),
-                       (i ? 'A' : 'X') );
 
-            if ( l_errl )
-            {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
+            const TARGETING::ATTR_IO_XBUS_MASTER_MODE_type l_master_mode =
+                l_PbusConnection.first->getAttr
+                <TARGETING::ATTR_IO_XBUS_MASTER_MODE>();
 
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
+            TARGETING::ATTR_FAPI_NAME_type  l_master_target_name = {0};
+            TARGETING::ATTR_FAPI_NAME_type  l_slave_target_name = {0};
+            if (l_master_mode == fapi2::ENUM_ATTR_IO_XBUS_MASTER_MODE_TRUE)
+            {
+                l_linktrain_failures = run_linktraining(
+                                            l_thisPbusFapi2Target,
+                                            l_connectedPbusFapi2Target,
+                                            l_StepError);
+                fapi2::toString(l_thisPbusFapi2Target,
+                                l_master_target_name,
+                                sizeof(l_master_target_name));
+                fapi2::toString(l_connectedPbusFapi2Target,
+                                l_slave_target_name,
+                                sizeof(l_slave_target_name));
+            }
+            else
+            {
+                l_linktrain_failures = run_linktraining(
+                                            l_connectedPbusFapi2Target,
+                                            l_thisPbusFapi2Target,
+                                            l_StepError);
+                fapi2::toString(l_connectedPbusFapi2Target,
+                                l_master_target_name,
+                                sizeof(l_master_target_name));
+                fapi2::toString(l_thisPbusFapi2Target,
+                                l_slave_target_name,
+                                sizeof(l_slave_target_name));
+            }
 
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
+            if (l_linktrain_failures)
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%d attempts to linktrain XBUS targets failed. "
+                    "Master: %s, Slave: %s", l_linktrain_failures,
+                    l_master_target_name, l_slave_target_name);
+            }
+            else
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "Successfully linktrained XBUS targets. "
+                    "Master: %s, Slave: %s",
+                    l_master_target_name, l_slave_target_name);
             }
         }
-    }
+    } // end for MaxBusSet
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_io_run_training exit" );
-*/
+
     // end task, returning any errorlogs to IStepDisp
     return l_StepError.getErrorHandle();
 }
+
+
+
+
+/**
+ *  @brief Try to train link between target's endpoints
+ *
+ *  @param[in]  i_master_target  Master XBUS target endpoint
+ *  @param[in]  i_slave_target   Slave  XBUS target endpoint
+ *  @param[out] o_step_error   Failing error logs added to this
+ *
+ *  @pre Target services must be initialized
+ *  @post See "return"
+ *  @return uint8_t Number of failures (maximum failures = 4)
+ *  @retval 4 Means all attempts to train links failed
+ */
+uint8_t run_linktraining(
+                const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_master_target,
+                const fapi2::Target<fapi2::TARGET_TYPE_XBUS> &i_slave_target,
+                ISTEP_ERROR::IStepError & o_step_error)
+{
+    uint8_t o_failures = 0;
+    errlHndl_t l_errl = NULL;
+
+    // clock group is either 0 or 1
+    // need to train both groups and allow for them to differ
+    uint8_t l_this_group = 0;
+    uint8_t l_connected_group = 0;
+    uint8_t l_group_loop = 0;
+    for (l_group_loop = 0; l_group_loop < 4; l_group_loop++)
+    {
+        l_this_group = l_group_loop / 2;      // 0, 0, 1, 1
+        l_connected_group = l_group_loop % 2; // 0, 1, 1, 0
+
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                 "Running p9_io_xbus_linktrain HWP on "
+                 "master target %.8X group %d and "
+                 "slave target %.8X group %d.",
+                 TARGETING::get_huid(i_master_target),
+                 l_this_group,
+                 TARGETING::get_huid(i_slave_target),
+                 l_connected_group );
+
+        FAPI_INVOKE_HWP(l_errl,
+                        p9_io_xbus_linktrain,
+                        i_master_target,
+                        l_this_group,
+                        i_slave_target,
+                        l_connected_group);
+
+        if (l_errl)
+        {
+            o_failures++;
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                 "Failure #%d) ERROR 0x%.8X : p9_io_xbus_linktrain "
+                 "HWP on master target %.8X group %d and "
+                 "slave target %.8X group %d.",
+                 o_failures, l_errl->reasonCode(),
+                 TARGETING::get_huid(i_master_target),
+                 l_this_group,
+                 TARGETING::get_huid(i_slave_target),
+                 l_connected_group );
+
+            // capture the target data in the elog
+            ErrlUserDetailsTarget(i_master_target).addToLog( l_errl );
+            ErrlUserDetailsTarget(i_slave_target).addToLog( l_errl );
+
+            // Create IStep error log and cross ref error that occurred
+            o_step_error.addErrorDetails( l_errl);
+
+            // Commit Error
+            errlCommit(l_errl, HWPF_COMP_ID);
+            l_errl = NULL;
+        }
+    }
+    return o_failures;
+}
+
+
+
 };
diff --git a/src/usr/isteps/istep09/call_fabric_post_trainadv.C b/src/usr/isteps/istep09/call_fabric_post_trainadv.C
index 9fdfffa45..210427508 100644
--- a/src/usr/isteps/istep09/call_fabric_post_trainadv.C
+++ b/src/usr/isteps/istep09/call_fabric_post_trainadv.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,13 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include  <pbusLinkSvc.H>
+#include  <fapi2/target.H>
+#include  <fapi2/plat_hwp_invoker.H>
+
+// HWP
+#include    <p9_io_xbus_post_trainadv.H>
+
 namespace   ISTEP_09
 {
 
@@ -72,88 +79,107 @@ using   namespace   HWAS;
 void*    call_fabric_post_trainadv( void    *io_pArgs )
 {
     IStepError  l_StepError;
-    //@TODO RTC:134079
-/*    errlHndl_t  l_errl  =   NULL;
+    errlHndl_t  l_errl  =   NULL;
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_post_trainadv entry" );
 
-    TargetPairs_t l_PbusConnections;
-    TargetPairs_t::iterator l_itr;
-    const uint32_t MaxBusSet = 2;
-    TYPE busSet[MaxBusSet] = { TYPE_ABUS, TYPE_XBUS };
-
-    // Note:
-    // Due to lab tester board environment, HW procedure writer (Varkey) has
-    // requested to send in one target of a time (we used to send in
-    // both ends in one call). Even though they don't have to be
-    // in order, we should keep the pair concept here in case we need to send
-    // in a pair in the future again.
+    EDI_EI_INITIALIZATION::TargetPairs_t l_PbusConnections;
+    const uint32_t MaxBusSet = 1;
+    TYPE busSet[MaxBusSet] = { TYPE_XBUS }; // TODO RTC:152304 - add TYPE_OBUS
 
     for (uint32_t ii = 0; (!l_errl) && (ii < MaxBusSet); ii++)
     {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                            l_PbusConnections, busSet[ii] );
-
-        for (l_itr = l_PbusConnections.begin();
-             l_itr != l_PbusConnections.end();
-             ++l_itr)
+        l_errl = EDI_EI_INITIALIZATION::PbusLinkSvc::getTheInstance().
+                    getPbusConnections(l_PbusConnections, busSet[ii]);
+        if (l_errl)
         {
-            const fapi::Target l_fapi_endp1_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->first)));
-            const fapi::Target l_fapi_endp2_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->second)));
-
-            //@TODO RTC:133830  call the HWP with each bus connection
-            //FAPI_INVOKE_HWP( l_errl, p9_io_post_trainadv,
-            //l_fapi_endp1_target );
-
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "%s : %cbus connection fabric_post_trainadv. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->first) );
-            if ( l_errl )
-            {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
+                "ERROR 0x%.8X : getPbusConnections TYPE_%cBUS returns error",
+                l_errl->reasonCode(), (ii ? 'X':'O') );
 
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
+            // Create IStep error log and cross reference to error that occurred
+            l_StepError.addErrorDetails( l_errl );
 
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
+            // Commit the error log
+            // Log should be deleted and set to NULL in errlCommit.
+            errlCommit(l_errl, HWPF_COMP_ID);
 
-            //@TODO RTC:133830 call the HWP with each bus connection
-            FAPI_INVOKE_HWP( l_errl, io_post_trainadv, l_fapi_endp2_target );
+            // Don't continue with a potential bad connection set
+            break;
+        }
 
-            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                  "%s : %cbus connection fabric_post_trainadv. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->second) );
-            if ( l_errl )
+        for (const auto & l_PbusConnection: l_PbusConnections)
+        {
+            const TARGETING::Target* l_thisPbusTarget = l_PbusConnection.first;
+            const TARGETING::Target* l_connectedPbusTarget =
+                                                    l_PbusConnection.second;
+
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_thisPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_thisPbusTarget)));
+
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_connectedPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_connectedPbusTarget)));
+
+            // group is either 0 or 1,
+            // need to train both groups and allow for them to differ
+            uint8_t l_this_group = 0;
+            uint8_t l_connected_group = 0;
+            uint8_t l_group_loop = 0;
+            for (l_group_loop = 0; l_group_loop < 4; l_group_loop++)
             {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
-
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
-
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
-        }
-    }
+                l_this_group = l_group_loop / 2;      // 0, 0, 1, 1
+                l_connected_group = l_group_loop % 2; // 0, 1, 1, 0
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                         "Running p9_io_xbus_post_trainadv HWP on "
+                         "this %cbus target %.8X (group %d) and connected "
+                         "target %.8X (group %d)",
+                         (ii ? 'X' : 'O'),
+                         TARGETING::get_huid(l_thisPbusTarget), l_this_group,
+                         TARGETING::get_huid(l_connectedPbusTarget),
+                         l_connected_group );
+
+                FAPI_INVOKE_HWP( l_errl, p9_io_xbus_post_trainadv,
+                                 l_thisPbusFapi2Target, l_this_group,
+                                 l_connectedPbusFapi2Target,
+                                 l_connected_group );
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%s : %cbus connection p9_io_xbus_post_trainadv "
+                    "Target 0x%.8X using group %d, connected target 0x%.8X "
+                    "using group %d",
+                    (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'O'),
+                    TARGETING::get_huid(l_thisPbusTarget), l_this_group,
+                    TARGETING::get_huid(l_connectedPbusTarget),
+                    l_connected_group );
+
+                if ( l_errl )
+                {
+                    // capture the target data in the elog
+                    ErrlUserDetailsTarget(l_thisPbusTarget).addToLog(l_errl);
+                    ErrlUserDetailsTarget(l_connectedPbusFapi2Target).addToLog
+                        (l_errl);
+
+                    // Create IStep error log and cross ref error that occurred
+                    l_StepError.addErrorDetails( l_errl );
+
+                    // Commit Error
+                    errlCommit( l_errl, HWPF_COMP_ID );
+                    // We want to continue the training despite the error, so
+                    // no break
+                    l_errl = NULL;
+                }
+
+            } // end of groups
+        } // end of connection TYPE combinations
+    } // end of connection set loop
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_post_trainadv exit" );
-*/
+
     // end task, returning any errorlogs to IStepDisp
     return l_StepError.getErrorHandle();
 }
diff --git a/src/usr/isteps/istep09/call_fabric_pre_trainadv.C b/src/usr/isteps/istep09/call_fabric_pre_trainadv.C
index 39ad83b14..d3c8df3bc 100644
--- a/src/usr/isteps/istep09/call_fabric_pre_trainadv.C
+++ b/src/usr/isteps/istep09/call_fabric_pre_trainadv.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,13 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include  <pbusLinkSvc.H>
+#include  <fapi2/target.H>
+#include  <fapi2/plat_hwp_invoker.H>
+
+// HWP
+#include    <p9_io_xbus_pre_trainadv.H>
+
 namespace   ISTEP_09
 {
 
@@ -73,88 +80,106 @@ using   namespace   HWAS;
 void*    call_fabric_pre_trainadv( void    *io_pArgs )
 {
     IStepError  l_StepError;
-    //@TODO RTC:134079
-/*
     errlHndl_t  l_errl  =   NULL;
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_pre_trainadv entry" );
 
-    TargetPairs_t l_PbusConnections;
-    TargetPairs_t::iterator l_itr;
-    const uint32_t MaxBusSet = 2;
-    TYPE busSet[MaxBusSet] = { TYPE_ABUS, TYPE_XBUS };
-
-    // Note:
-    // Due to lab tester board environment, HW procedure writer (Varkey) has
-    // requested to send in one target of a time (we used to send in
-    // both ends in one call). Even though they don't have to be
-    // in order, we should keep the pair concept here in case we need to send
-    // in a pair in the future again.
+    EDI_EI_INITIALIZATION::TargetPairs_t l_PbusConnections;
+    const uint32_t MaxBusSet = 1;
+    TYPE busSet[MaxBusSet] = { TYPE_XBUS }; // TODO RTC:152304 - add TYPE_OBUS
 
     for (uint32_t ii = 0; (!l_errl) && (ii < MaxBusSet); ii++)
     {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                            l_PbusConnections, busSet[ii] );
-
-        for (l_itr = l_PbusConnections.begin();
-             l_itr != l_PbusConnections.end();
-             ++l_itr)
+        l_errl = EDI_EI_INITIALIZATION::PbusLinkSvc::getTheInstance().
+                    getPbusConnections(l_PbusConnections, busSet[ii]);
+        if (l_errl)
         {
-            const fapi::Target l_fapi_endp1_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->first)));
-            const fapi::Target l_fapi_endp2_target(
-                   (ii ? TARGET_TYPE_XBUS_ENDPOINT : TARGET_TYPE_ABUS_ENDPOINT),
-                   (const_cast<TARGETING::Target*>(l_itr->second)));
-
-            //@TODO RTC:133830  call the HWP with each bus connection
-            //FAPI_INVOKE_HWP( l_errl, p9_io_pre_trainadv,
-            //l_fapi_endp1_target );
-
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "%s : %cbus connection fabric_pre_trainadv. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->first) );
-            if ( l_errl )
-            {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->first).addToLog( l_errl );
+                "ERROR 0x%.8X : getPbusConnections TYPE_%cBUS returns error",
+                l_errl->reasonCode(), (ii ? 'X':'O') );
 
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
+            // Create IStep error log and cross reference to error that occurred
+            l_StepError.addErrorDetails( l_errl );
 
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
+            // Commit the error log
+            // Log should be deleted and set to NULL in errlCommit.
+            errlCommit(l_errl, HWPF_COMP_ID);
 
-            //@TODO RTC:133830  call the HWP with each bus connection
-            FAPI_INVOKE_HWP( l_errl, io_pre_trainadv, l_fapi_endp2_target );
+            // Don't continue with a potential bad connection set
+            break;
+        }
 
-            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                   "%s : %cbus connection fabric_pre_trainadv. Target 0x%.8X",
-                       (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'A'),
-                        TARGETING::get_huid(l_itr->second) );
-            if ( l_errl )
+        for (const auto & l_PbusConnection: l_PbusConnections)
+        {
+            const TARGETING::Target* l_thisPbusTarget = l_PbusConnection.first;
+            const TARGETING::Target* l_connectedPbusTarget =
+                                                    l_PbusConnection.second;
+
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_thisPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_thisPbusTarget)));
+
+            const fapi2::Target <fapi2::TARGET_TYPE_XBUS>
+                l_connectedPbusFapi2Target(
+                (const_cast<TARGETING::Target*>(l_connectedPbusTarget)));
+
+            // group is either 0 or 1,
+            // need to train both groups and allow for them to differ
+            uint8_t l_this_group = 0;
+            uint8_t l_connected_group = 0;
+            uint8_t l_group_loop = 0;
+            for (l_group_loop = 0; l_group_loop < 4; l_group_loop++)
             {
-                // capture the target data in the elog
-                ErrlUserDetailsTarget(l_itr->second).addToLog( l_errl );
-
-                // Create IStep error log and cross ref error that occurred
-                l_StepError.addErrorDetails( l_errl );
-
-                // Commit Error
-                errlCommit( l_errl, HWPF_COMP_ID );
-                // We want to continue the training despite the error, so
-                // no break
-            }
-        }
-    }
+                l_this_group = l_group_loop / 2;      // 0, 0, 1, 1
+                l_connected_group = l_group_loop % 2; // 0, 1, 1, 0
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                         "Running p9_io_xbus_pre_trainadv HWP on "
+                         "this %cbus target %.8X (group %d) and connected "
+                         "target %.8X (group %d)",
+                         (ii ? 'X' : 'O'),
+                         TARGETING::get_huid(l_thisPbusTarget), l_this_group,
+                         TARGETING::get_huid(l_connectedPbusTarget),
+                         l_connected_group );
+
+                FAPI_INVOKE_HWP( l_errl, p9_io_xbus_pre_trainadv,
+                                 l_thisPbusFapi2Target, l_this_group,
+                                 l_connectedPbusFapi2Target,
+                                 l_connected_group );
+
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                    "%s : %cbus connection p9_io_xbus_pre_trainadv "
+                    "Target 0x%.8X using group %d, connected target 0x%.8X "
+                    "using group %d",
+                    (l_errl ? "ERROR" : "SUCCESS"), (ii ? 'X' : 'O'),
+                    TARGETING::get_huid(l_thisPbusTarget), l_this_group,
+                    TARGETING::get_huid(l_connectedPbusTarget),
+                    l_connected_group );
+
+                if ( l_errl )
+                {
+                    // capture the target data in the elog
+                    ErrlUserDetailsTarget(l_thisPbusTarget).addToLog(l_errl);
+                    ErrlUserDetailsTarget(l_connectedPbusTarget).addToLog
+                        (l_errl);
+
+                    // Create IStep error log and cross ref error that occurred
+                    l_StepError.addErrorDetails( l_errl );
+
+                    // Commit Error
+                    errlCommit( l_errl, HWPF_COMP_ID );
+                    // We want to continue the training despite the error, so
+                    // no break
+                    l_errl = NULL;
+                }
+
+            } // end of groups
+        } // end of connection TYPE combinations
+    } // end of connection set loop
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_fabric_pre_trainadv exit" );
-*/
+
     // end task, returning any errorlogs to IStepDisp
     return l_StepError.getErrorHandle();
 }
diff --git a/src/usr/isteps/istep09/call_proc_fab_iovalid.C b/src/usr/isteps/istep09/call_proc_fab_iovalid.C
index 1b79ea55b..866f03eaf 100644
--- a/src/usr/isteps/istep09/call_proc_fab_iovalid.C
+++ b/src/usr/isteps/istep09/call_proc_fab_iovalid.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,13 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include <fapi2/target.H>
+#include <fapi2/plat_hwp_invoker.H>
+#include <errl/errlmanager.H>
+
+// HWP
+#include <p9_fab_iovalid.H>
+
 namespace   ISTEP_09
 {
 
@@ -71,130 +78,48 @@ using   namespace   HWAS;
 void*    call_proc_fab_iovalid( void    *io_pArgs )
 {
     IStepError l_StepError;
-    //@TODO RTC:134079 port to fapi2
-/*    errlHndl_t l_errl = NULL;
+    errlHndl_t l_errl = NULL;
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_proc_fab_iovalid entry" );
 
-    // Get all chip/chiplet targets
+    //
+    //  get a list of all the procs in the system
+    //
     TARGETING::TargetHandleList l_cpuTargetList;
     getAllChips(l_cpuTargetList, TYPE_PROC);
 
-    TargetPairs_t l_abusConnections;
-    TargetPairs_t l_xbusConnections;
-    l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                 l_abusConnections, TYPE_ABUS, false );
-    if (!l_errl)
-    {
-        l_errl = PbusLinkSvc::getTheInstance().getPbusConnections(
-                                 l_xbusConnections, TYPE_XBUS, false );
-    }
-
-    if ( l_errl )
+    // Loop through all processors including master
+    for (const auto & l_cpu_target: l_cpuTargetList)
     {
-        // Create IStep error log and cross reference error that occurred
-        l_StepError.addErrorDetails( l_errl );
-
-        // Commit Error
-        errlCommit( l_errl, HWPF_COMP_ID );
-    }
-
-    std::vector<proc_fab_iovalid_proc_chip> l_smp;
+        const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>l_fapi2_proc_target(
+                  l_cpu_target);
 
-    for (TargetHandleList::const_iterator l_cpu_iter = l_cpuTargetList.begin();
-         l_StepError.isNull() && (l_cpu_iter != l_cpuTargetList.end());
-         ++l_cpu_iter)
-    {
-        proc_fab_iovalid_proc_chip l_procEntry;
-
-        TARGETING::TargetHandle_t l_pTarget = *l_cpu_iter;
-        fapi::Target l_fapiproc_target( TARGET_TYPE_PROC_CHIP, l_pTarget);
-
-        l_procEntry.this_chip = l_fapiproc_target;
-        l_procEntry.a0 = false;
-        l_procEntry.a1 = false;
-        l_procEntry.a2 = false;
-        l_procEntry.x0 = false;
-        l_procEntry.x1 = false;
-        l_procEntry.x2 = false;
-        l_procEntry.x3 = false;
-
-        TARGETING::TargetHandleList l_abuses;
-        getChildChiplets( l_abuses, l_pTarget, TYPE_ABUS );
-
-        for (TargetHandleList::const_iterator l_abus_iter = l_abuses.begin();
-            l_abus_iter != l_abuses.end();
-            ++l_abus_iter)
-        {
-            TARGETING::TargetHandle_t l_target = *l_abus_iter;
-            uint8_t l_srcID = l_target->getAttr<ATTR_CHIP_UNIT>();
-            TargetPairs_t::iterator l_itr = l_abusConnections.find(l_target);
-            if ( l_itr == l_abusConnections.end() )
-            {
-                continue;
-            }
-            switch (l_srcID)
-            {
-                case 0: l_procEntry.a0 = true; break;
-                case 1: l_procEntry.a1 = true; break;
-                case 2: l_procEntry.a2 = true; break;
-               default: break;
-            }
-        }
-
-        TARGETING::TargetHandleList l_xbuses;
-        getChildChiplets( l_xbuses, l_pTarget, TYPE_XBUS );
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                 "Running p9_fab_iovalid HWP on processor target %.8X",
+                 TARGETING::get_huid(l_cpu_target) );
 
-        for (TargetHandleList::const_iterator l_xbus_iter = l_xbuses.begin();
-            l_xbus_iter != l_xbuses.end();
-            ++l_xbus_iter)
+        FAPI_INVOKE_HWP(l_errl, p9_fab_iovalid, l_fapi2_proc_target, true);
+        if(l_errl)
         {
-            TARGETING::TargetHandle_t l_target = *l_xbus_iter;
-            uint8_t l_srcID = l_target->getAttr<ATTR_CHIP_UNIT>();
-            TargetPairs_t::iterator l_itr = l_xbusConnections.find(l_target);
-            if ( l_itr == l_xbusConnections.end() )
-            {
-                continue;
-            }
-            switch (l_srcID)
-            {
-                case 0: l_procEntry.x0 = true; break;
-                case 1: l_procEntry.x1 = true; break;
-                case 2: l_procEntry.x2 = true; break;
-                case 3: l_procEntry.x3 = true; break;
-               default: break;
-            }
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                     "ERROR 0x%.8X : p9_fab_iovalid "
+                     "HWP returns error for HUID %.8X",
+                     l_errl->reasonCode(),
+                     TARGETING::get_huid(l_cpu_target) );
+
+            // capture the target data in the elog
+            ErrlUserDetailsTarget(l_cpu_target).addToLog( l_errl );
+
+            l_StepError.addErrorDetails(l_errl);
+            errlCommit(l_errl, HWPF_COMP_ID);
+            l_errl = NULL;
         }
-
-        l_smp.push_back(l_procEntry);
-    }
-
-    if (!l_errl)
-    {
-        //@TODO RTC:133830
-        //FAPI_INVOKE_HWP( l_errl, p9_fab_iovalid, l_smp, true );
-
-        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "%s : proc_fab_iovalid HWP.",
-                (l_errl ? "ERROR" : "SUCCESS"));
-    }
-
-    if (l_errl)
-    {
-        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                "ERROR : call_proc_fab_iovalid encountered an error");
-
-        // Create IStep error log and cross reference error that occurred
-        l_StepError.addErrorDetails( l_errl );
-
-        // Commit Error
-        errlCommit( l_errl, HWPF_COMP_ID );
-    }
+    } // end of going through all processors//
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_proc_fab_iovalid exit" );
-*/
+
     // end task, returning any errorlogs to IStepDisp
     return l_StepError.getErrorHandle();
 }
diff --git a/src/usr/isteps/istep09/call_proc_smp_link_layer.C b/src/usr/isteps/istep09/call_proc_smp_link_layer.C
index 106d59be9..e0d36556f 100644
--- a/src/usr/isteps/istep09/call_proc_smp_link_layer.C
+++ b/src/usr/isteps/istep09/call_proc_smp_link_layer.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -57,6 +57,12 @@
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/common/trace.H>
 
+#include <fapi2/target.H>
+#include <fapi2/plat_hwp_invoker.H>
+#include <errl/errlmanager.H>
+
+#include <p9_smp_link_layer.H>
+
 namespace   ISTEP_09
 {
 
@@ -77,14 +83,38 @@ void*   call_proc_smp_link_layer( void    *io_pArgs )
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_proc_smp_link_layer entry" );
-    //@TODO RTC:133830 call HWP
-    //FAPI_INVOKE_HWP(l_errl,p9_smp_link_layer);
-    if(l_errl)
+
+    //
+    //  get a list of all the procs in the system
+    //
+    TARGETING::TargetHandleList l_cpuTargetList;
+    getAllChips(l_cpuTargetList, TYPE_PROC);
+
+    for (const auto & l_cpu_target: l_cpuTargetList)
     {
-        l_StepError.addErrorDetails(l_errl);
-        errlCommit(l_errl, HWPF_COMP_ID);
+        const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>
+                l_fapi2_proc_target (l_cpu_target);
+
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                     "Running p9_smp_link_layer HWP on processor target %.8X",
+                     TARGETING::get_huid(l_cpu_target) );
+
+        FAPI_INVOKE_HWP(l_errl,p9_smp_link_layer, l_fapi2_proc_target);
+        if(l_errl)
+        {
+          TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                     ERR_MRK "call_proc_smp_link_layer> Failed HWP call, "
+                     " HUID = 0x%08X",
+                     TARGETING::get_huid(l_cpu_target) );
+            l_StepError.addErrorDetails(l_errl);
+            errlCommit(l_errl, HWPF_COMP_ID);
+        }
     }
-    return l_StepError.getErrorHandle();
 
+    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+               "call_proc_smp_link_layer exit" );
+
+    return l_StepError.getErrorHandle();
 }
+
 };
diff --git a/src/usr/isteps/istep09/makefile b/src/usr/isteps/istep09/makefile
index f24a5a317..5e3280ff7 100644
--- a/src/usr/isteps/istep09/makefile
+++ b/src/usr/isteps/istep09/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2015
+# Contributors Listed Below - COPYRIGHT 2015,2016
 # [+] International Business Machines Corp.
 #
 #
@@ -23,8 +23,17 @@
 #
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../../../..
+PROCEDURES_PATH    = ${ROOTPATH}/src/import/chips/p9/procedures
 MODULE = istep09
 
+EXTRAINCDIR += ${PROCEDURES_PATH}/hwp/nest/
+EXTRAINCDIR += ${PROCEDURES_PATH}/hwp/io/
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/utils/
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
+EXTRAINCDIR += ${ROOTPATH}/src/import/hwpf/fapi2/include/
+EXTRAINCDIR += ${ROOTPATH}/src/include/usr/fapi2/
+EXTRAINCDIR += ${ROOTPATH}/src/usr/isteps/
+
 OBJS += call_fabric_erepair.o
 OBJS += call_fabric_io_dccal.o
 OBJS += call_fabric_pre_trainadv.o
@@ -35,6 +44,33 @@ OBJS += call_host_startprd_pbus.o
 OBJS += call_host_attnlisten_proc.o
 OBJS += call_proc_fab_iovalid.o
 
-EXTRAINCDIR += ${ROOTPATH}/src/include/usr/ecmddatabuffer/
+VPATH += ${PROCEDURES_PATH}/hwp/nest/ ${PROCEDURES_PATH}/hwp/io/
+
+include ${ROOTPATH}/procedure.rules.mk
+
+# 9.1 fabric_erepair
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_restore_erepair.mk
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_pdwn_lanes.mk
+
+# 9.2 fabric_io_dccal
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_dccal.mk
+
+# 9.3 fabric_pre_trainadv
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_pre_trainadv.mk
+
+# 9.4 fabric_io_run_training
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_linktrain.mk
+
+# 9.5 fabric_post_trainadv
+include ${PROCEDURES_PATH}/hwp/io/p9_io_xbus_post_trainadv.mk
+
+# 9.6 proc_smp_link_layer : Start SMP link layer
+include ${PROCEDURES_PATH}/hwp/nest/p9_smp_link_layer.mk
+
+# 9.7 & 9.8 - no hwp
+
+# 9.9 p9_fab_iovalid
+include ${PROCEDURES_PATH}/hwp/nest/p9_fab_iovalid.mk
+
 
 include ${ROOTPATH}/config.mk