Adds MREP training for LRDIMM

Change-Id: Ic119a3e474acc2d2c5db3ed0b4e35828c0ea9812
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/67178
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Dev-Ready: STEPHEN GLANCY <sglancy@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/67182
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>

5 files changed, 797 insertions, 19 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H b/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
index 7862377dc..3d013b307 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
@@ -215,7 +215,6 @@ enum
     STAT_HUNG = 0x0ull,
 };
 
-
 ///
 /// @class instruction_t
 /// @brief Class for ccs instructions
@@ -494,6 +493,30 @@ inline instruction_t<T> des_command()
 }
 
 ///
+/// @brief Create, initialize a NTTM read CCS command
+/// @tparam T the target type of the chiplet which executes the CCS instruction
+/// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
+/// @return the Device Deselect CCS instruction
+/// @note need to setup 4 cycles delay
+///
+template< fapi2::TargetType T, typename TT = ccsTraits<T> >
+inline instruction_t<T> nttm_read_command()
+{
+    // setup 4 cycles delay
+    constexpr uint64_t NTTM_READ_DELAY = 4;
+    // No.. there's no field for this one. it's an undocumented bit
+    constexpr uint64_t NTTM_MODE_FORCE_READ = 33;
+
+    // get the des_command
+    auto l_command = des_command<T>();
+    // set to CCS_INST_ARR1 register
+    l_command.arr1.template setBit<NTTM_MODE_FORCE_READ>();
+    l_command.arr1.template insertFromRight<TT::ARR1_IDLES, TT::ARR1_IDLES_LEN>(NTTM_READ_DELAY);
+
+    return l_command;
+}
+
+///
 /// @brief Create, initialize a JEDEC Device Power Down Entry CCS command
 /// @tparam T the target type of the chiplet which executes the CCS instruction
 /// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
@@ -997,6 +1020,30 @@ inline fapi2::ReturnCode write_mode( const fapi2::Target<T>& i_target, const fap
 }
 
 ///
+/// @brief config the NTTM
+/// @tparam T the target type of the chiplet which executes the CCS instruction
+/// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
+/// @param[in] i_mcbist the target to operate
+/// @param[in] i_nttm_mode NTTM we need to turn on or off (i.e. ON, OFF)
+/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS if ok
+///
+template< fapi2::TargetType T, typename TT = ccsTraits<T> >
+inline fapi2::ReturnCode configure_nttm( const fapi2::Target<T>& i_target,
+        const mss::states i_nttm_mode)
+{
+    fapi2::buffer<uint64_t> l_data;
+
+    FAPI_TRY(read_mode(i_target, l_data));
+
+    l_data.writeBit<TT::NTTM_MODE>(i_nttm_mode);
+
+    FAPI_TRY(write_mode(i_target, l_data));
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
 /// @brief Execute a set of CCS instructions - multiple ports
 /// @tparam T the target type of the chiplet which executes the CCS instruction
 /// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pba.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pba.C
index dff5e102f..6a49c1391 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pba.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pba.C
@@ -201,6 +201,9 @@ fapi2::ReturnCode execute_commands( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>
     {
         ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
 
+        // Inserts the DES command to ensure we keep our CKE high
+        l_program.iv_instructions.push_back(mss::ccs::des_command<fapi2::TARGET_TYPE_MCBIST>());
+
         // Makes a copy of the vector, so we can do the function space swaps correctly
         auto l_bcws = i_bcws;
         FAPI_TRY(insert_function_space_select(l_bcws));
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/phy/dp16.C b/src/import/chips/p9/procedures/hwp/memory/lib/phy/dp16.C
index a23478ff9..c7b557d9a 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/phy/dp16.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/phy/dp16.C
@@ -4483,8 +4483,11 @@ fapi2::ReturnCode write_force_dq_capture( const fapi2::Target<fapi2::TARGET_TYPE
         const mss::states i_state)
 {
     typedef dp16Traits<TARGET_TYPE_MCA> TT;
-
     fapi2::buffer<uint64_t> l_data;
+
+    // read the data from one of the RD_DIA_CONFIG5 registers
+    // because the value should be the same for other registers
+    FAPI_TRY(mss::getScom(i_target, MCA_DDRPHY_DP16_RD_DIA_CONFIG5_P0_0, l_data));
     l_data.writeBit<TT::FORCE_FIFO_CAPTURE>(i_state);
 
     FAPI_TRY( mss::scom_blastah(i_target, TT::RD_DIA_CONFIG5_REG, l_data) );
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.C b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.C
index e7c100874..0707bf696 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.C
@@ -45,6 +45,8 @@
 #include <lib/dimm/ddr4/control_word_ddr4.H>
 #include <lib/dimm/ddr4/data_buffer_ddr4.H>
 #include <lib/workarounds/ccs_workarounds.H>
+#include <lib/ccs/ccs.H>
+#include <lib/mc/port.H>
 
 namespace mss
 {
@@ -151,40 +153,425 @@ fapi2::ReturnCode mpr_pattern_wr_rank(const fapi2::Target<fapi2::TARGET_TYPE_MCA
     FAPI_TRY( ccs::execute(l_mcbist,
                            l_program,
                            i_target) );
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+//
+/// @brief Does a CCS NTTM mode read
+/// @param[in] i_target - the MCA target on which to operate
+/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS if ok
+///
+fapi2::ReturnCode mrep::execute_nttm_mode_read(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target) const
+{
+    mss::ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+
+    // Note: CKE are enabled by default in the NTTM mode read command, so we should be good to go
+    // set the NTTM read mode
+    l_program.iv_instructions.push_back(mss::ccs::nttm_read_command<fapi2::TARGET_TYPE_MCBIST>());
+
+    // turn on NTTM mode
+    FAPI_TRY( mss::ccs::configure_nttm(l_mcbist, mss::states::ON),
+              "%s failed to turn on NTTM mode", mss::c_str(i_target) );
+
+    // Issue CCS
+    FAPI_TRY(ccs::execute( l_mcbist,
+                           l_program,
+                           i_target) );
+    // turn off NTTM mode
+    FAPI_TRY( mss::ccs::configure_nttm(l_mcbist, mss::states::OFF),
+              "%s failed to turn off NTTM mode", mss::c_str(i_target) );
 
 fapi_try_exit:
     return fapi2::current_err;
 }
 
 ///
-/// @brief Sets up and runs the calibration step
+/// @brief Executes the pre-cal step workaround
 /// @param[in] i_target - the MCA target on which to operate
 /// @param[in] i_rp - the rank pair
 /// @param[in] i_abort_on_error - whether or not we are aborting on cal error
 /// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
 ///
-fapi2::ReturnCode mrep::run( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
-                             const uint64_t i_rp,
-                             const uint8_t i_abort_on_error ) const
+fapi2::ReturnCode mrep::pre_workaround( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                        const uint64_t i_rp,
+                                        const uint8_t i_abort_on_error ) const
 {
-    return fapi2::FAPI2_RC_SUCCESS;
+    FAPI_INF("%s setting up the read pointer enable rp%u", mss::c_str(i_target), i_rp);
+    FAPI_TRY( mss::setup_read_pointer_delay(i_target));
+
+    // call function to force DQ capture in Read FIFO to support DDR4 LRDIMM calibration.
+    FAPI_TRY( mss::dp16::write_force_dq_capture(i_target, mss::states::ON),
+              "%s failed to write force dq capture", mss::c_str(i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
 }
 
 ///
-/// @brief Executes a cal step with workarounds
+/// @brief Executes the post-cal step workaround
 /// @param[in] i_target - the MCA target on which to operate
 /// @param[in] i_rp - the rank pair
 /// @param[in] i_abort_on_error - whether or not we are aborting on cal error
-/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
+/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS if ok
+///
+fapi2::ReturnCode mrep::post_workaround( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+        const uint64_t i_rp,
+        const uint8_t i_abort_on_error ) const
+{
+    // call function to force DQ capture in Read FIFO to support DDR4 LRDIMM calibration.
+    FAPI_TRY( mss::dp16::write_force_dq_capture(i_target, mss::states::OFF),
+              "%s failed to write exit dq capture", mss::c_str(i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Write the results to buffer generate PBA commands
+/// @param[in] i_target the DIMM target
+/// @param[in] i_rank the rank number
+/// @param[in] i_mrep_result a vector of the MREP result
+/// @param[out] o_container the PBA commands structure
+/// @return FAPI2_RC_SUCCESS if and only if ok
+/// @note a little helper to allow us to unit test that we generate the PBA commands ok
 ///
-fapi2::ReturnCode mrep::execute( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
-                                 const uint64_t i_rp,
-                                 const uint8_t i_abort_on_error ) const
+fapi2::ReturnCode mrep::write_result_to_buffers_helper( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+        const uint8_t i_rank,
+        const std::vector<std::pair<recorder, recorder>>& i_mrep_result,
+        mss::ddr4::pba::commands& o_container) const
 {
+    uint8_t l_buffer = 0;
+    // Clears out the PBA container to ensure we don't issue undesired commands
+    o_container.clear();
+
+    // Looops through and generates the PBA commands
+    for(const auto& l_recorder : i_mrep_result)
+    {
+        const auto l_result_nibble0 = l_recorder.first.iv_final_delay;
+        const auto l_result_nibble1 = l_recorder.second.iv_final_delay;
+
+        // Function space is derived from the rank
+        // 2 is for Nibble 0, 3 is for Nibble 1
+        // Data corresponds to the final setting we have
+        // Delay is for PBA, bumping it way out so we don't have issues
+        constexpr uint64_t PBA_DELAY = 255;
+        constexpr uint64_t BCW_NIBBLE0 = 0x02;
+        constexpr uint64_t BCW_NIBBLE1 = 0x03;
+
+        const mss::cw_info MREP_FINAL_SET_BCW_N0( i_rank,
+                BCW_NIBBLE0,
+                l_result_nibble0,
+                PBA_DELAY,
+                mss::CW8_DATA_LEN,
+                mss::cw_info::BCW);
+        const mss::cw_info MREP_FINAL_SET_BCW_N1( i_rank,
+                BCW_NIBBLE1,
+                l_result_nibble1,
+                PBA_DELAY,
+                mss::CW8_DATA_LEN,
+                mss::cw_info::BCW);
+
+        // Each buffer contains two nibbles
+        // Each nibble corresponds to one BCW
+        // Add in the buffer control words
+        FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N0));
+        FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N1));
+
+        ++l_buffer;
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief write the result to buffer
+/// @param[in] i_target the DIMM target
+/// @param[in] i_rank the rank number
+/// @param[in] i_mrep_result a vector of the MREP results
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode mrep::write_result_to_buffers( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+        const uint8_t i_rank,
+        const std::vector<std::pair<recorder, recorder>>& i_mrep_result) const
+{
+    mss::ddr4::pba::commands l_container;
+
+    // Sets up the PBA commands
+    FAPI_TRY(write_result_to_buffers_helper( i_target,
+             i_rank,
+             i_mrep_result,
+             l_container),
+             "%s rank%u failed generating PBA commands",
+             mss::c_str(i_target), i_rank);
+
+    // Issue the PBA to set the final MREP results
+    FAPI_TRY(mss::ddr4::pba::execute_commands(l_container));
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief analyze with each nibble
+/// @param[in] i_target the MCA target
+/// @param[in] i_result_nibble the result need to analyze
+/// @param[in] i_buffer the buffer number
+/// @param[in] i_delay the delay we set
+/// @param[in] i_nibble the nibble number
+/// @param[in, out] io_recorder we need to get and record
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode mrep::analyze_result_for_each_nibble( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+        const uint8_t i_result_nibble,
+        const uint8_t i_buffer,
+        const uint8_t i_delay,
+        const uint8_t i_nibble,
+        recorder& io_recorder ) const
+{
+    switch(i_result_nibble)
+    {
+        case 0x00:
+            io_recorder.iv_seen0 = true;
+            break;
+
+        case 0xF0:
+
+            // We need to see a 0 prior to a 1 for a 0 to 1 transition
+            if(io_recorder.iv_seen0)
+            {
+                io_recorder.iv_seen1 = true;
+            }
+
+            break;
+
+        default:
+            FAPI_ERR( "%s Get DQS value 0x%02x not 0 or 0xF from buffer %x nibble %x for delay 0x%02x",
+                      mss::c_str(i_target), i_result_nibble, i_buffer, i_nibble, i_delay);
+            break;
+    }
+
+    // Record the 0->1 transition only if:
+    // 1) we've seen a 0
+    // 2) we've seen a 1
+    // 3) we have not recorded a value prior (don't want to overwrite our good values) (not 0)
+    if( (io_recorder.iv_seen0 == true) &&
+        (io_recorder.iv_seen1 == true) &&
+        (io_recorder.iv_final_delay == 0) )
+    {
+        io_recorder.iv_final_delay = i_delay;
+        FAPI_DBG( "%s buffer:%u nibble:%u found a 0->1 transition at delay 0x%02x",
+                  mss::c_str(i_target), i_buffer, i_nibble, i_delay );
+    }
+
     return fapi2::FAPI2_RC_SUCCESS;
 }
 
 ///
+/// @brief analyze the result of MREP
+/// @param[in] i_target the MCA target
+/// @param[in] i_delay the delay number we current set
+/// @param[in, out] io_recorders a vector of the MREP results
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode mrep::analyze_mrep_result( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+        const uint8_t i_delay,
+        std::vector<std::pair<recorder, recorder>>& io_recorders) const
+{
+    constexpr uint8_t NIBBLE0 = 0;
+    constexpr uint8_t NIBBLE1 = 1;
+    constexpr uint8_t MASK_NIBBLE0 = 0xf0;
+    constexpr uint8_t MASK_NIBBLE1 = 0x0f;
+    data_response l_data;
+    uint8_t l_buffer = 0;
+
+    FAPI_TRY( l_data.read(i_target),
+              "%s failed to read MREP data response delay:0x%02x",
+              mss::c_str(i_target),
+              i_delay );
+
+    // Note: we want to update the value of the results recorder, so no const
+    for(auto& l_recorder : io_recorders)
+    {
+        // All beats should be the same, until proven otherwise, just use beat 0
+        constexpr uint64_t DEFAULT_BEAT = 0;
+        const uint8_t l_buffer_result = l_data.iv_buffer_beat[l_buffer][DEFAULT_BEAT];
+        const auto l_result_nibble0 = l_buffer_result & MASK_NIBBLE0;
+        const auto l_result_nibble1 = (l_buffer_result & MASK_NIBBLE1) << BITS_PER_NIBBLE;
+
+        FAPI_DBG( "%s delay:0x%02x MREP result buffer:%u data:0x%02x N0:0x%02x N1:0x%02x",
+                  mss::c_str(i_target), i_delay,
+                  l_buffer, l_buffer_result,
+                  l_result_nibble0, l_result_nibble1);
+
+        // Temporary variables for some beautification below
+        auto& l_recorder_nibble0 = l_recorder.first;
+        auto& l_recorder_nibble1 = l_recorder.second;
+
+        FAPI_TRY(analyze_result_for_each_nibble( i_target,
+                 l_result_nibble0,
+                 l_buffer,
+                 i_delay,
+                 NIBBLE0,
+                 l_recorder_nibble0) );
+        FAPI_TRY(analyze_result_for_each_nibble( i_target,
+                 l_result_nibble1,
+                 l_buffer,
+                 i_delay,
+                 NIBBLE1,
+                 l_recorder_nibble1) );
+
+        l_buffer++;
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Sets MREP Delay value
+/// @param[in] i_target the DIMM target
+/// @param[in] i_rank the rank to operate on - drives the function space select
+/// @param[in] delay value /64 Tck - MREP delay value
+/// @return FAPI2_RC_SUCCESS if okay
+/// @note Sets DA setting for buffer control word (F[3:0]BC2x, F[3:0]BC3x)
+///
+fapi2::ReturnCode mrep::set_delay(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                  const uint8_t i_rank,
+                                  const uint8_t i_delay ) const
+{
+    mss::ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+    const auto& l_mca = mss::find_target<fapi2::TARGET_TYPE_MCA>(i_target);
+
+    std::vector<cw_info> l_bcws =
+    {
+        {i_rank, NIBBLE0_BCW_NUMBER, i_delay, mss::tmrc(), mss::CW8_DATA_LEN, cw_info::BCW},
+        {i_rank, NIBBLE1_BCW_NUMBER, i_delay, mss::tmrc(), mss::CW8_DATA_LEN, cw_info::BCW},
+    };
+
+    uint8_t l_sim = 0;
+    FAPI_TRY(mss::is_simulation(l_sim));
+
+    // Ensure our CKE's are powered on
+    l_program.iv_instructions.push_back(mss::ccs::des_command<fapi2::TARGET_TYPE_MCBIST>());
+
+    // Inserts the function space selects
+    FAPI_TRY(mss::ddr4::insert_function_space_select(l_bcws));
+
+    // Sets up the CCS instructions
+    FAPI_TRY(control_word_engine(i_target,
+                                 l_bcws,
+                                 l_sim,
+                                 l_program.iv_instructions));
+
+    // Make sure we leave everything powered on
+    mss::ccs::workarounds::hold_cke_high(l_program.iv_instructions);
+
+    // Issue CCS
+    FAPI_TRY( ccs::execute(l_mcbist,
+                           l_program,
+                           l_mca) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Sets up and runs the calibration step
+/// @param[in] i_target - the MCA target on which to operate
+/// @param[in] i_rp - the rank pair
+/// @param[in] i_abort_on_error - whether or not we are aborting on cal error
+/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS if ok
+///
+fapi2::ReturnCode mrep::run( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                             const uint64_t i_rp,
+                             const uint8_t i_abort_on_error ) const
+{
+    constexpr uint8_t MPR_LOCATION0 = 0;
+    std::vector<uint64_t> l_ranks;
+    uint8_t l_rank_index = 0;
+    FAPI_INF("%s RP%d starting calibrate MREP", mss::c_str(i_target), i_rp);
+
+    const auto& l_dimms = mss::find_targets<fapi2::TARGET_TYPE_DIMM>(i_target);
+
+    FAPI_TRY( mss::rank::get_ranks_in_pair( i_target, i_rp, l_ranks),
+              "Failed get_ranks_in_pair in mrep::run %s",
+              mss::c_str(i_target) );
+
+    // Loops over all ranks within this rank pair
+    // MREP is a buffer to DRAM calibration step, so we need to calibrate all ranks seperately
+    for (const auto& l_rank : l_ranks)
+    {
+        // Skip over invalid ranks (NO_RANK)
+        if(l_rank == NO_RANK)
+        {
+            FAPI_DBG("%s RP%u l_rank_index:%u is being skipped as it's not configured (%u)",
+                     mss::c_str(i_target), i_rp, l_rank_index, l_rank);
+            ++l_rank_index;
+            continue;
+        }
+
+        FAPI_DBG("%s RP%u rank index number %u has rank %u", mss::c_str(i_target), i_rp, l_rank_index, l_rank);
+        const auto& l_dimm = l_dimms[mss::rank::get_dimm_from_rank(l_rank)];
+
+        // Vector represents the number of LRDIMM buffers
+        // The pair represents the two nibbles that we need to calibrate within the buffer
+        std::vector<std::pair<recorder, recorder>> l_results_recorder(MAX_LRDIMM_BUFFERS);
+
+        // 1) Put the buffer in MREP mode -> host issues BCW's
+        FAPI_TRY(set_buffer_training(l_dimm, ddr4::MREP), "%s failed set_buffer_training", mss::c_str(l_dimm));
+
+        // 2) Gets the ranks on which to put DRAM into MPR mode
+        FAPI_TRY( mpr_load(l_dimm, fapi2::ENUM_ATTR_EFF_MPR_MODE_ENABLE, l_rank), "%s failed mpr_load rank%u",
+                  mss::c_str(l_dimm), l_rank);
+
+        // Loop through all of our delays
+        for(uint8_t l_delay = 0; l_delay < MREP_MAX_DELAY; ++l_delay)
+        {
+            // 3) Set the MREP l_delay -> host issues BCW's
+            FAPI_TRY(set_delay(l_dimm, l_rank, l_delay), "%s failed set_delay rank%u delay%u", mss::c_str(l_dimm), l_rank, l_delay);
+
+            // 4) Do an MPR read -> host issues RD command to the DRAM
+            FAPI_TRY( mpr_read(l_dimm, MPR_LOCATION0, l_rank), "%s failed mpr_read rank%u delay%u", mss::c_str(l_dimm),
+                      l_rank, l_delay);
+
+            // 4.1) Do an NTTM mode read -> forces the logic to read out the data
+            FAPI_TRY(execute_nttm_mode_read(i_target));
+
+            // 5) Analyze the results -> host/FW read from CCS results and go
+            FAPI_TRY( analyze_mrep_result(i_target, l_delay, l_results_recorder), "%s failed analyze_mrep_result rank%u delay%u",
+                      mss::c_str(l_dimm), l_rank, l_delay);
+        } //l_delay loop
+
+        // 6) Error check -> if we had stuck 1 or stuck 0 (never saw a 0 to 1 or a 1 to 0 transition) exit out with an error
+        FAPI_TRY(error_check(l_dimm, l_rank, l_results_recorder), "%s failed error_check rank:%u", mss::c_str(l_dimm), l_rank);
+
+        // 7) Apply MREP offset to ranks based upon tCK RD preamble mode
+        FAPI_TRY(apply_final_offset(l_dimm, l_results_recorder), "%s failed apply_final_offset rank%u", mss::c_str(l_dimm),
+                 l_rank);
+
+        // 8) take DRAM out of MPR -> host issues MRS
+        FAPI_TRY( mpr_load(l_dimm, fapi2::ENUM_ATTR_EFF_MPR_MODE_DISABLE, l_rank), "%s failed mpr_load %u", mss::c_str(l_dimm),
+                  l_rank);
+
+        // 9) take the buffer out of MREP mode -> host issues BCW's
+        FAPI_TRY(set_buffer_training(l_dimm, ddr4::NORMAL), "%s failed set_buffer_training", mss::c_str(l_dimm));
+
+        // 10) Write final values into the buffers -> host issues BCW's in PBA mode (values are calculated in step 7)
+        FAPI_TRY( write_result_to_buffers(l_dimm, l_rank, l_results_recorder), "%s failed write_result_to_buffers rank%u",
+                  mss::c_str(l_dimm), l_rank);
+
+        l_rank_index++;
+    }//l_rank loop
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
 /// @brief Calculates the number of cycles a given calibration step will take
 /// @param[in] i_target - the MCA target on which to operate
 /// @return l_cycles - the number of cycles a given calibration step wil take
@@ -315,7 +702,9 @@ uint64_t mwd::calculate_cycles( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_t
 /// @param[in,out] io_cal_steps - the bit mask of calibration steps
 /// @return a vector of the calibration steps to run
 ///
-void deconfigure_steps(const uint8_t i_dimm_type, const bool i_sim, fapi2::buffer<uint32_t>& io_cal_steps)
+void deconfigure_steps(const uint8_t i_dimm_type,
+                       const bool i_sim,
+                       fapi2::buffer<uint32_t>& io_cal_steps)
 {
     // If the DIMM type is an LRDIMM, configure for LRDIMM
     if(i_dimm_type == fapi2::ENUM_ATTR_EFF_DIMM_TYPE_LRDIMM)
@@ -338,7 +727,6 @@ void deconfigure_steps(const uint8_t i_dimm_type, const bool i_sim, fapi2::buffe
     .clearBit<HWL>();
 }
 
-
 } // ns lrdimm
 
 } // ns training
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.H b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.H
index 4e0e92735..0e940af3e 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.H
@@ -38,6 +38,7 @@
 #ifndef MSS_LRDIMM_TRAINING_H
 #define MSS_LRDIMM_TRAINING_H
 
+#include <generic/memory/lib/utils/shared/mss_generic_consts.H>
 #include <lib/phy/mss_training.H>
 #include <lib/ccs/ccs.H>
 #include <lib/dimm/ddr4/mrs_load_ddr4.H>
@@ -45,6 +46,9 @@
 #include <lib/dimm/ddr4/data_buffer_ddr4.H>
 #include <lib/phy/seq.H>
 #include <generic/memory/lib/utils/buffer_ops.H>
+#include <lib/mcbist/mcbist.H>
+#include <lib/dimm/ddr4/pba.H>
+#include <lib/workarounds/ccs_workarounds.H>
 
 namespace mss
 {
@@ -197,6 +201,56 @@ fapi2::ReturnCode mpr_pattern_wr_rank(const fapi2::Target<fapi2::TARGET_TYPE_MCA
                                       const uint64_t i_rank,
                                       const uint32_t i_pattern);
 
+/// @class data_response
+/// @brief A structure that stores the results from the CCS on a per-buffer per-beat level
+///
+struct data_response
+{
+    static constexpr uint64_t MAX_NUM_BEATS = 8;
+    uint8_t iv_buffer_beat[MAX_LRDIMM_BUFFERS][MAX_NUM_BEATS];
+
+    ///
+    /// @brief Reads the data response for the LRDIMM
+    /// @param[in] i_target the MCA target on which to operate
+    /// @return fapi2::ReturnCode SUCCESS iff code exectutes successfully
+    ///
+    fapi2::ReturnCode read(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target)
+    {
+        // Magic number is
+        // 1) read from the read buffer
+        // 2) read from the magic CCS start addres
+        // 3) auto-increment the addresses to read from
+        constexpr uint64_t CCS_READ_CONFIGURE = 0x7c20000000000000;
+        FAPI_TRY(mss::putScom(i_target, MCA_WREITE_AACR, CCS_READ_CONFIGURE));
+
+        // Now read out all 8 beats
+        for(uint8_t l_beat = 0; l_beat < MAX_NUM_BEATS; ++l_beat)
+        {
+            // Read out the data
+            fapi2::buffer<uint64_t> l_data;
+            FAPI_TRY(mss::getScom(i_target, MCA_AADR, l_data));
+
+            // Parse the data out into the buffers for this beat
+            // Each data buffer is 8 bits wide, so we parse each buffer by 8 bits
+            l_data.extractToRight<0, BITS_PER_BYTE>(iv_buffer_beat[0][l_beat])
+            .extractToRight< 8, BITS_PER_BYTE>(iv_buffer_beat[1][l_beat])
+            .extractToRight<16, BITS_PER_BYTE>(iv_buffer_beat[2][l_beat])
+            .extractToRight<24, BITS_PER_BYTE>(iv_buffer_beat[3][l_beat])
+            .extractToRight<32, BITS_PER_BYTE>(iv_buffer_beat[4][l_beat])
+            .extractToRight<40, BITS_PER_BYTE>(iv_buffer_beat[5][l_beat])
+            .extractToRight<48, BITS_PER_BYTE>(iv_buffer_beat[6][l_beat])
+            .extractToRight<56, BITS_PER_BYTE>(iv_buffer_beat[7][l_beat]);
+
+            // Read out the ECC buffer
+            FAPI_TRY(mss::getScom(i_target, MCA_AAER, l_data));
+            l_data.extractToRight<0, BITS_PER_BYTE>(iv_buffer_beat[8][l_beat]);
+        }
+
+    fapi_try_exit:
+        return fapi2::current_err;
+    }
+};
+
 // TK:LRDIMM Do we need separate coarse vs fine steps?
 ///
 /// @brief MREP training step
@@ -204,6 +258,29 @@ fapi2::ReturnCode mpr_pattern_wr_rank(const fapi2::Target<fapi2::TARGET_TYPE_MCA
 class mrep : public step
 {
     public:
+        // Per the LRDIMM spec, the MREP training values have one cycle per 64 ticks
+        static constexpr uint8_t MREP_MAX_DELAY = 64;
+        static constexpr uint8_t NIBBLE0_BCW_NUMBER = 0x02;
+        static constexpr uint8_t NIBBLE1_BCW_NUMBER = 0x03;
+
+        ///
+        /// @class mrep_decoder
+        /// @brief A structure that stores the record of DQS value
+        ///
+        struct recorder
+        {
+            // With MREP, we need to see a 0 to 1 transition on each nibble
+            // This indicates that the MDQS and the clock are aligned
+            // Our error cases, therefore, need to check for not seeing a 0 or not seeing a 1
+            // We want to keep track of if we've seen a 0 and if we've seen a 1
+            // We also want to note where our 0 to 1 transition occurs
+            // We can set the value in at the end of the training algorithm
+            bool iv_seen0;
+            bool iv_seen1;
+            uint8_t iv_final_delay;
+        };
+
+
         mrep() :
             step("MREP")
         {}
@@ -214,6 +291,18 @@ class mrep : public step
         ~mrep() = default;
 
         ///
+        /// @brief Executes the pre-cal step workaround
+        /// @param[in] i_target - the MCA target on which to operate
+        /// @param[in] i_rp - the rank pair
+        /// @param[in] i_abort_on_error - whether or not we are aborting on cal error
+        /// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
+        /// TODO:RTC200368 Update MREP for RAS - add in unit tests for this funtion
+        ///
+        fapi2::ReturnCode pre_workaround( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                          const uint64_t i_rp,
+                                          const uint8_t i_abort_on_error ) const;
+
+        ///
         /// @brief Sets up and runs the calibration step
         /// @param[in] i_target - the MCA target on which to operate
         /// @param[in] i_rp - the rank pair
@@ -225,15 +314,16 @@ class mrep : public step
                                const uint8_t i_abort_on_error ) const;
 
         ///
-        /// @brief Executes a cal step with workarounds
+        /// @brief Executes the post-cal step workaround
         /// @param[in] i_target - the MCA target on which to operate
         /// @param[in] i_rp - the rank pair
         /// @param[in] i_abort_on_error - whether or not we are aborting on cal error
         /// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
+        /// TODO:RTC200368 Update MREP for RAS - add in unit tests for this funtion
         ///
-        fapi2::ReturnCode execute( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
-                                   const uint64_t i_rp,
-                                   const uint8_t i_abort_on_error ) const;
+        fapi2::ReturnCode post_workaround( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                           const uint64_t i_rp,
+                                           const uint8_t i_abort_on_error ) const;
 
         ///
         /// @brief Calculates the number of cycles a given calibration step will take
@@ -241,6 +331,177 @@ class mrep : public step
         /// @return l_cycles - the number of cycles a given calibration step wil take
         ///
         uint64_t calculate_cycles( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target ) const;
+
+        ///
+        /// @brief write the result to buffer
+        /// @param[in] i_target the DIMM target
+        /// @param[in] i_rank the rank number
+        /// @param[in] i_mrep_result a vector of the MREP result
+        /// @return FAPI2_RC_SUCCESS if and only if ok
+        ///
+        fapi2::ReturnCode write_result_to_buffers( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                const uint8_t i_rank,
+                const std::vector<std::pair<recorder, recorder>>& i_mrep_result) const;
+
+        ///
+        /// @brief Write the results to buffer generate PBA commands
+        /// @param[in] i_target the DIMM target
+        /// @param[in] i_rank the rank number
+        /// @param[in] i_mrep_result a vector of the MREP result
+        /// @param[out] o_container the PBA commands structure
+        /// @return FAPI2_RC_SUCCESS if and only if ok
+        /// @note a little helper to allow us to unit test that we generate the PBA commands ok
+        ///
+        fapi2::ReturnCode write_result_to_buffers_helper( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                const uint8_t i_rank,
+                const std::vector<std::pair<mss::training::lrdimm::mrep::recorder, mss::training::lrdimm::mrep::recorder>>&
+                i_mrep_result,
+                mss::ddr4::pba::commands& o_container) const;
+
+        ///
+        ///// @brief Does a CCS NTTM mode read
+        ///// @param[in] i_target - the MCA target on which to operate
+        ///// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS if ok
+        /////
+        fapi2::ReturnCode execute_nttm_mode_read(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target) const;
+
+        ///
+        /// @brief analyze with each nibble
+        /// @param[in] i_target the MCA target
+        /// @param[in] i_result_nibble the result need to analyze
+        /// @param[in] i_buffer the buffer number
+        /// @param[in] i_delay the delay we set
+        /// @param[in] i_nibble the nibble number
+        /// @param[in, out] io_recorder we need to get and record
+        /// @return FAPI2_RC_SUCCESS if and only if ok
+        ///
+        fapi2::ReturnCode analyze_result_for_each_nibble( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                const uint8_t i_result_nibble,
+                const uint8_t i_buffer,
+                const uint8_t i_delay,
+                const uint8_t i_nibble,
+                recorder& io_recorder ) const;
+
+        ///
+        /// @brief analyze the result of MREP
+        /// @param[in] i_target the MCA target
+        /// @param[in] i_delay the delay number we current set
+        /// @param[in, out] io_recorders a vector of the MREP results
+        /// @return FAPI2_RC_SUCCESS if and only if ok
+        ///
+        fapi2::ReturnCode analyze_mrep_result( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                               const uint8_t i_delay,
+                                               std::vector<std::pair<recorder, recorder>>& io_recorders) const;
+
+        ///
+        /// @brief Sets MREP Delay value
+        /// @param[in] i_target the DIMM target
+        /// @param[in] i_rank the rank to operate on - drives the function space select
+        /// @param[in] delay value /64 Tck - MREP delay value
+        /// @return FAPI2_RC_SUCCESS if okay
+        /// @note Sets DA setting for buffer control word (F[3:0]BC2x, F[3:0]BC3x)
+        ///
+        fapi2::ReturnCode set_delay(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                    const uint8_t i_rank,
+                                    const uint8_t i_delay ) const;
+
+        ///
+        /// @brief error check helper during the MREP training
+        /// @param[in] i_target the DIMM target
+        /// @param[in] i_rank the rank to operate on
+        /// @param[in] i_buffer the buffer to operate on
+        /// @param[in] i_nibble the nibble to operate on
+        /// @param[in] i_recorder the recorders for error check
+        /// @return FAPI2_RC_SUCCESS if okay
+        /// @note Check each nibble to ensure seen0/seen1 are true
+        ///
+        inline fapi2::ReturnCode error_check_helper(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                const uint8_t i_rank,
+                const uint8_t i_buffer,
+                const uint8_t i_nibble,
+                const recorder& i_recorder) const
+        {
+            if( (true != i_recorder.iv_seen0)
+                || (true != i_recorder.iv_seen1))
+            {
+                FAPI_ERR("%s rank%u buffer%u nibble%u had an error %s a 0 %s a 1",
+                         mss::c_str(i_target), i_rank, i_buffer, i_nibble,
+                         i_recorder.iv_seen0 ? "saw" : "did not see",
+                         i_recorder.iv_seen1 ? "saw" : "did not see");
+            }
+
+            return fapi2::FAPI2_RC_SUCCESS;
+        }
+
+        ///
+        /// @brief error check during the MREP training
+        /// @param[in] i_target the DIMM target
+        /// @param[in] i_rank the rank to operate on
+        /// @param[in] i_recorders the recorders for error check
+        /// @return FAPI2_RC_SUCCESS if okay
+        /// @note Check each nibble to ensure seen0/seen1 are true
+        ///
+        inline fapi2::ReturnCode error_check(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                             const uint8_t i_rank,
+                                             const std::vector<std::pair<recorder, recorder>>& i_recorders) const
+        {
+            uint8_t l_buffer = 0;
+            constexpr uint8_t NIBBLE0 = 0;
+            constexpr uint8_t NIBBLE1 = 1;
+
+            // Loop for all buffer results, check the results to ensure seen0/seen1 are true
+            for(auto& l_recorder : i_recorders)
+            {
+                // Temporary variables
+                auto& l_recorder_nibble0 = l_recorder.first;
+                auto& l_recorder_nibble1 = l_recorder.second;
+
+                FAPI_TRY(error_check_helper(i_target, i_rank, l_buffer, NIBBLE0, l_recorder_nibble0));
+                FAPI_TRY(error_check_helper(i_target, i_rank, l_buffer, NIBBLE1, l_recorder_nibble1));
+
+                l_buffer++;
+            }
+
+        fapi_try_exit:
+            return fapi2::current_err;
+        }
+
+        ///
+        /// @brief Apply MREP offset to ranks based upon tCK RD preamble mode
+        /// @param[in] i_target the DIMM target
+        /// @param[in,out] io_results the recorders for error check
+        /// @return FAPI2_RC_SUCCESS if okay
+        /// @note Check each nibble to ensure seen0/seen1 are true
+        ///
+        inline fapi2::ReturnCode apply_final_offset(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                std::vector<std::pair<recorder, recorder>>& io_results) const
+        {
+            // Per the LRDIMM spec, we need to add an offset of 32 for 1 tCK and 64 for 2 tCK
+            constexpr uint8_t OFFSET_1TCK = 32;
+            constexpr uint8_t OFFSET_2TCK = 64;
+            uint8_t l_tck = 0;
+            uint8_t l_buffer = 0;
+
+            // try to get rd preamble
+            FAPI_TRY(mss::eff_rd_preamble(i_target, l_tck));
+
+            // Loop for all buffer results, apply the rd pramble
+            for(auto& l_result : io_results)
+            {
+                const auto OFFSET = l_tck == fapi2::ENUM_ATTR_EFF_RD_PREAMBLE_1NCLK ? OFFSET_1TCK : OFFSET_2TCK;
+
+                // Could make this math into a function - that way we can test it
+                l_result.first.iv_final_delay = (l_result.first.iv_final_delay + OFFSET) % MREP_MAX_DELAY;
+                l_result.second.iv_final_delay = (l_result.second.iv_final_delay + OFFSET) % MREP_MAX_DELAY;
+                FAPI_DBG("%s buffer:%u final values for nibble0:0x%02x nibble1:0x%02x",
+                         mss::c_str(i_target), l_buffer, l_result.first.iv_final_delay, l_result.second.iv_final_delay);
+                l_buffer++;
+            }
+
+        fapi_try_exit:
+            return fapi2::current_err;
+        }
+
 };
 
 ///
@@ -383,13 +644,89 @@ class mwd : public step
 // TK:LRDIMM Identify if Host Interface Write training Training is any different
 
 ///
+/// @brief Sets data buffer training mode control word
+/// @param[in] i_target the DIMM target
+/// @param[in] i_mode buffer training mode
+/// @return FAPI2_RC_SUCCESS iff ok
+/// @note Sets buffer control word (BC0C) setting
+///
+inline fapi2::ReturnCode set_buffer_training(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+        const mss::ddr4::training i_mode )
+{
+
+    mss::ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+    const auto& l_mca = mss::find_target<fapi2::TARGET_TYPE_MCA>(i_target);
+
+    FAPI_TRY(mss::ddr4::set_buffer_training(i_target, i_mode, l_program.iv_instructions));
+
+    FAPI_TRY( ccs::execute(l_mcbist,
+                           l_program,
+                           l_mca));
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Makes CCS instruction to set MPR Mode
+/// @param[in] i_target a DIMM target
+/// @param[in] i_mode setting for MPR mode
+/// @param[in] i_rank DIMM rank
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+inline fapi2::ReturnCode mpr_load(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                  const uint8_t i_mode,
+                                  const uint64_t i_rank)
+{
+    mss::ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+    const auto& l_mca = mss::find_target<fapi2::TARGET_TYPE_MCA>(i_target);
+
+    FAPI_TRY( mss::ddr4::mpr_load(i_target,
+                                  i_mode,
+                                  i_rank,
+                                  l_program.iv_instructions) );
+
+    FAPI_TRY( ccs::execute(l_mcbist,
+                           l_program,
+                           l_mca) );
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Makes CCS instruction for an MPR read
+/// @param[in] i_target a DIMM target
+/// @param[in] i_mode MPR location
+/// @param[in] i_rank DIMM rank
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+inline fapi2::ReturnCode mpr_read( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                   const uint64_t i_mpr_loc,
+                                   const uint64_t i_rank)
+{
+    mss::ccs::program<fapi2::TARGET_TYPE_MCBIST> l_program;
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+    const auto& l_mca = mss::find_target<fapi2::TARGET_TYPE_MCA>(i_target);
+
+    FAPI_TRY( ddr4::mpr_read<fapi2::TARGET_TYPE_MCBIST>(i_target, i_mpr_loc, i_rank, l_program.iv_instructions));
+    FAPI_TRY( ccs::execute(l_mcbist,
+                           l_program,
+                           l_mca) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+///
 /// @brief Deconfigures calibration steps depending upon LRDIMM type
 /// @param[in] i_dimm_type - DIMM type
 /// @param[in] i_sim - simulation mode or not
 /// @param[in,out] io_cal_steps - the bit mask of calibration steps
 /// @return a vector of the calibration steps to run
 ///
-void deconfigure_steps(const uint8_t i_dimm_type, const bool i_sim, fapi2::buffer<uint32_t>& io_cal_steps);
+void deconfigure_steps(const uint8_t i_dimm_type,
+                       const bool i_sim,
+                       fapi2::buffer<uint32_t>& io_cal_steps);
 
 } // ns training