Adds STR entry and exit functions to support NVDIMM

Change-Id: Ia4385d885c4a4dd23378ec488a93209f547cb20b
CQ:SW420342
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/55226
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/55986
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

8 files changed, 438 insertions, 43 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
index 37d0d8d8a..306b01362 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
@@ -22,3 +22,277 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
+
+///
+/// @file nvdimm_utils.C
+/// @brief Subroutines to support nvdimm backup/restore process
+///
+// *HWP HWP Owner: Tsung Yeung <tyeung@us.ibm.com>
+// *HWP HWP Backup: Stephen Glancy <sglancy@us.ibm.com>
+// *HWP Team: Memory
+// *HWP Level: 3
+// *HWP Consumed by: FSP:HB
+
+#include <fapi2.H>
+#include <vector>
+
+#include <lib/dimm/ddr4/nvdimm_utils.H>
+#include <lib/mc/mc.H>
+#include <lib/ccs/ccs.H>
+#include <lib/dimm/rank.H>
+#include <lib/mss_attribute_accessors.H>
+#include <lib/utils/poll.H>
+#include <lib/utils/count_dimm.H>
+#include <lib/mcbist/address.H>
+#include <lib/mcbist/memdiags.H>
+#include <lib/mcbist/mcbist.H>
+#include <lib/mcbist/settings.H>
+#include <lib/utils/conversions.H>
+
+#include <lib/mc/port.H>
+#include <lib/phy/dp16.H>
+#include <lib/dimm/rcd_load.H>
+#include <lib/dimm/mrs_load.H>
+#include <lib/dimm/ddr4/pda.H>
+#include <lib/dimm/ddr4/zqcal.H>
+
+using fapi2::TARGET_TYPE_MCBIST;
+using fapi2::TARGET_TYPE_MCA;
+using fapi2::TARGET_TYPE_DIMM;
+
+namespace mss
+{
+
+namespace nvdimm
+{
+
+///
+/// @brief Helper for self_refresh_exit(). Uses memdiag to read the port to force
+/// CKE back to high. Stolen from mss_lab_memdiags.C
+/// Specialization for TARGET_TYPE_MCA
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template< >
+fapi2::ReturnCode self_refresh_exit_helper( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target )
+{
+
+    const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
+    fapi2::buffer<uint64_t> l_status;
+
+    // A small vector of addresses to poll during the polling loop
+    const std::vector<mss::poll_probe<fapi2::TARGET_TYPE_MCBIST>> l_probes =
+    {
+        {l_mcbist, "mcbist current address", MCBIST_MCBMCATQ},
+    };
+
+    // We'll fill in the initial delay below
+    // Heuristically defined and copied from the f/w version of memdiags
+    mss::poll_parameters l_poll_parameters(0, 200, 100 * mss::DELAY_1MS, 200, 500);
+    uint64_t l_memory_size = 0;
+
+    FAPI_TRY( mss::eff_memory_size(l_mcbist, l_memory_size) );
+    l_poll_parameters.iv_initial_delay = mss::calculate_initial_delay(l_mcbist, (l_memory_size * mss::BYTES_PER_GB));
+
+    {
+        // Force this to run on the targeted port only
+        const auto& l_port = mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_target);
+        mss::mcbist::address l_start;
+        mss::mcbist::end_boundary l_end_boundary = mss::mcbist::end_boundary::STOP_AFTER_SLAVE_RANK;
+        l_start.set_port(l_port);
+        mss::mcbist::stop_conditions l_stop_conditions;
+
+        // Read with super fast read
+        // Set up with mcbist target, stop conditions above
+        // Using defaults for starting at first valid address and stop at end of slave rank
+        FAPI_TRY ( mss::memdiags::sf_read(l_mcbist,
+                                          l_stop_conditions,
+                                          l_start,
+                                          l_end_boundary,
+                                          l_start) );
+
+        bool l_poll_results = mss::poll(l_mcbist, MCBIST_MCBISTFIRQ, l_poll_parameters,
+                                        [&l_status](const size_t poll_remaining,
+                                                const fapi2::buffer<uint64_t>& stat_reg) -> bool
+        {
+            FAPI_DBG("mcbist firq 0x%llx, remaining: %d", stat_reg, poll_remaining);
+            l_status = stat_reg;
+            return l_status.getBit<MCBIST_MCBISTFIRQ_MCBIST_PROGRAM_COMPLETE>() == true;
+        },
+        l_probes);
+
+        FAPI_DBG("memdiags_read poll result: %d", l_poll_results);
+    }
+
+    return fapi2::FAPI2_RC_SUCCESS;
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Put target into self-refresh
+/// Specialization for TARGET_TYPE_MCA
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template<>
+fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target )
+{
+    fapi2::buffer<uint64_t> l_mbarpc0_data, l_mbastr0_data;
+
+    // Step 1 - In MBARPC0Q, disable power domain control, set domain to MAXALL_MINALL,
+    //          and enable minimum domain reduction
+    FAPI_TRY(mss::mc::read_mbarpc0(i_target, l_mbarpc0_data));
+    mss::mc::set_power_control_min_max_domains_enable( l_mbarpc0_data, mss::states::OFF );
+    mss::mc::set_power_control_min_max_domains( l_mbarpc0_data, mss::min_max_domains::MAXALL_MINALL );
+    mss::mc::set_power_control_min_domain_reduction_enable( l_mbarpc0_data, mss::states::ON );
+    FAPI_TRY(mss::mc::write_mbarpc0(i_target, l_mbarpc0_data));
+
+    // Step 2 - In MBASTR0Q, enable STR entry
+    FAPI_TRY(mss::mc::read_mbastr0(i_target, l_mbastr0_data));
+    mss::mc::set_self_time_refresh_enable( l_mbastr0_data, mss::states::ON );
+    FAPI_TRY(mss::mc::write_mbastr0(i_target, l_mbastr0_data));
+
+    // Step 3 - In MBARPC0Q, enable power domain control.
+    mss::mc::set_power_control_min_max_domains_enable( l_mbarpc0_data, mss::states::ON );
+    FAPI_TRY(mss::mc::write_mbarpc0(i_target, l_mbarpc0_data));
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Take the target out of self-refresh and restart refresh
+/// Specialization for TARGET_TYPE_MCA
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template< >
+fapi2::ReturnCode self_refresh_exit( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target )
+{
+    fapi2::buffer<uint64_t> l_mbarpc0_data, l_mbastr0_data;
+
+    // Step 1 - In MBARPC0Q, disable power domain control
+    FAPI_TRY(mss::mc::read_mbarpc0(i_target, l_mbarpc0_data));
+    mss::mc::set_power_control_min_max_domains_enable( l_mbarpc0_data, mss::states::OFF );
+    FAPI_TRY(mss::mc::write_mbarpc0(i_target, l_mbarpc0_data));
+
+    // Step 2 - In MBASTR0Q, disable STR entry
+    FAPI_TRY(mss::mc::read_mbastr0(i_target, l_mbastr0_data));
+    mss::mc::set_self_time_refresh_enable( l_mbastr0_data, mss::states::OFF );
+    FAPI_TRY(mss::mc::write_mbastr0(i_target, l_mbastr0_data));
+
+    // Step 3 - Run memdiags to read the port to force CKE back to high
+    FAPI_TRY(self_refresh_exit_helper(i_target));
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief PDA support for post restore transition
+/// Specialization for TARGET_TYPE_DIMM
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template<>
+fapi2::ReturnCode pda_vref_latch( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target )
+{
+    std::vector<uint64_t> l_ranks;
+    const auto& l_mca = mss::find_target<TARGET_TYPE_MCA>(i_target);
+    fapi2::buffer<uint8_t> l_value, l_range;
+    fapi2::ReturnCode l_rc(fapi2::FAPI2_RC_SUCCESS);
+
+    // Creates the MRS container class
+    mss::ddr4::pda::commands<mss::ddr4::mrs06_data> l_container;
+
+    // Get all the ranks in the dimm
+    mss::rank::ranks(i_target, l_ranks);
+
+    // Get the number of DRAMs
+    uint8_t l_width = 0;
+    mss::eff_dram_width(i_target, l_width);
+    const uint64_t l_num_drams = (l_width == fapi2::ENUM_ATTR_EFF_DRAM_WIDTH_X8) ? MAX_DRAMS_X8 : MAX_DRAMS_X4;
+
+    for (const auto& l_rank : l_ranks)
+    {
+
+        uint64_t l_rp = 0;
+        uint64_t l_wr_vref_value = 0;
+        bool l_wr_vref_range = 0;
+        fapi2::buffer<uint64_t> l_data ;
+
+        mss::rank::get_pair_from_rank(l_mca, l_rank, l_rp);
+
+        // create mrs06
+        mss::ddr4::mrs06_data l_mrs(i_target, l_rc);
+
+        // loop through all the dram
+        for(uint64_t l_dram = 0; l_dram < l_num_drams; l_dram++)
+        {
+            mss::dp16::wr_vref::read_wr_vref_register( l_mca, l_rp, l_dram, l_data);
+            mss::dp16::wr_vref::get_wr_vref_range( l_data, l_dram, l_wr_vref_range);
+            mss::dp16::wr_vref::get_wr_vref_value( l_data, l_dram, l_wr_vref_value);
+
+            l_mrs.iv_vrefdq_train_value[mss::index(l_rank)] = l_wr_vref_value;
+            l_mrs.iv_vrefdq_train_range[mss::index(l_rank)] = l_wr_vref_range;
+            l_container.add_command(i_target, l_rank, l_mrs, l_dram);
+        }
+    }
+
+    // Disable refresh
+    FAPI_TRY( mss::change_refresh_enable(l_mca, states::LOW) );
+
+    // execute_wr_vref_latch(l_container)
+    FAPI_TRY( mss::ddr4::pda::execute_wr_vref_latch(l_container) )
+
+    // Enable refresh
+    FAPI_TRY( mss::change_refresh_enable(l_mca, states::HIGH) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Post restore transition to support restoring nvdimm to
+/// a functional state after the restoring the data from NAND flash
+/// to DRAM
+/// Specialization for TARGET_TYPE_MCA
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template<>
+fapi2::ReturnCode post_restore_transition( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target )
+{
+    // Subseqent restore on later nvdimms would go wonky if this goes before STR exit...
+    FAPI_TRY( mss::rcd_load( i_target ) );
+
+    // Exit STR
+    FAPI_TRY( self_refresh_exit( i_target ) );
+
+    // Load the MRS
+    FAPI_TRY( mss::mrs_load( i_target ) );
+
+    // Do ZQCAL
+    {
+        fapi2::buffer<uint32_t> l_cal_steps_enabled;
+        l_cal_steps_enabled.setBit<mss::DRAM_ZQCAL>();
+
+        FAPI_DBG("cal steps enabled: 0x%08x ", l_cal_steps_enabled);
+        FAPI_TRY( mss::setup_and_execute_zqcal(i_target, l_cal_steps_enabled), "Error in nvdimm setup_and_execute_zqcal()" );
+    }
+
+    // Latch the trained PDA vref values for each dimm under the port
+    for (const auto& l_dimm : mss::find_targets<fapi2::TARGET_TYPE_DIMM>(i_target))
+    {
+        FAPI_TRY( pda_vref_latch( l_dimm ) );
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+}//ns nvdimm
+
+}//ns mss
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
index 03c1af7a0..b434abc3b 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
@@ -22,3 +22,75 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
+
+///
+/// @file nvdimm_utils.H
+/// @brief Subroutines to support nvdimm backup/restore process
+///
+// *HWP HWP Owner: Tsung Yeung <tyeung@us.ibm.com>
+// *HWP HWP Backup: Stephen Glancy <sglancy@us.ibm.com>
+// *HWP Team: Memory
+// *HWP Level: 3
+// *HWP Consumed by: FSP:HB
+
+#include <fapi2.H>
+#include <generic/memory/lib/utils/find.H>
+
+namespace mss
+{
+
+namespace nvdimm
+{
+
+///
+/// @brief Helper for self_refresh_exit().
+/// @tparam T the target type associated with this subroutine
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+/// @note Uses memdiag to read the port to force CKE back to high.
+/// Stolen from mss_lab_memdiags.C
+///
+template< fapi2::TargetType T >
+fapi2::ReturnCode self_refresh_exit_helper( const fapi2::Target<T>& i_target );
+
+///
+/// @brief Disable refresh and put target into self-refresh
+/// @tparam T the target type associated with this subroutine
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template< fapi2::TargetType T >
+fapi2::ReturnCode self_refresh_entry( const fapi2::Target<T>& i_target );
+
+///
+/// @brief Take the target out of self-refresh and restart refresh
+/// @tparam T the target type associated with this subroutine
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+template< fapi2::TargetType T >
+fapi2::ReturnCode self_refresh_exit( const fapi2::Target<T>& i_target );
+
+///
+/// @brief Latch write vref at per-dram basis
+/// @tparam T the target type associated with this subroutine
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+
+template< fapi2::TargetType T >
+fapi2::ReturnCode pda_vref_latch( const fapi2::Target<T>& i_target );
+
+///
+/// @brief Full post-restore transition for NVDIMM
+/// @tparam T the target type associated with this subroutine
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+
+template< fapi2::TargetType T >
+fapi2::ReturnCode post_restore_transition( const fapi2::Target<T>& i_target );
+
+}//ns nvdimm
+
+}//ns mss
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
index 661725928..609251cb1 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -49,13 +49,15 @@ using fapi2::FAPI2_RC_SUCCESS;
 namespace mss
 {
 ///
-/// @brief Perform the mrs_load operations - TARGET_TYPE_MCBIST specialization
-/// @param[in] i_target a fapi2::Target<TARGET_TYPE_MCBIST>
+/// @brief Perform the mrs_load operations - TARGET_TYPE_MCA specialization
+/// @param[in] i_target a fapi2::Target<TARGET_TYPE_MCA>
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
 template<>
-fapi2::ReturnCode mrs_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target )
+fapi2::ReturnCode mrs_load<TARGET_TYPE_MCA>( const fapi2::Target<TARGET_TYPE_MCA>& i_target )
 {
+    const auto& l_mcbist = mss::find_target<TARGET_TYPE_MCBIST>(i_target);
+
     // A vector of CCS instructions. We'll ask the targets to fill it, and then we'll execute it
     ccs::program<TARGET_TYPE_MCBIST> l_program;
 
@@ -64,18 +66,32 @@ fapi2::ReturnCode mrs_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_
     l_program.iv_poll.iv_initial_delay = 0;
     l_program.iv_poll.iv_initial_sim_delay = 0;
 
+    for ( const auto& d : find_targets<TARGET_TYPE_DIMM>(i_target) )
+    {
+        FAPI_DBG("mrs load for %s", mss::c_str(d));
+        FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) );
+    }
+
+    // We have to configure the CCS engine to let it know which port these instructions are
+    // going out (or whether it's broadcast ...) so lets execute the instructions we presently
+    // have so that we kind of do this by port
+    FAPI_TRY( ccs::execute(l_mcbist, l_program, i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Perform the mrs_load operations - TARGET_TYPE_MCBIST specialization
+/// @param[in] i_target a fapi2::Target<TARGET_TYPE_MCBIST>
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+template<>
+fapi2::ReturnCode mrs_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target )
+{
     for ( const auto& p : find_targets<TARGET_TYPE_MCA>(i_target) )
     {
-        for ( const auto& d : find_targets<TARGET_TYPE_DIMM>(p) )
-        {
-            FAPI_DBG("mrs load for %s", mss::c_str(d));
-            FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) );
-        }
-
-        // We have to configure the CCS engine to let it know which port these instructions are
-        // going out (or whether it's broadcast ...) so lets execute the instructions we presently
-        // have so that we kind of do this by port
-        FAPI_TRY( ccs::execute(i_target, l_program, p) );
+        FAPI_TRY( mrs_load(p) );
     }
 
 fapi_try_exit:
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load.C
index e165b71b7..128376e6e 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -49,15 +49,16 @@ using fapi2::FAPI2_RC_SUCCESS;
 
 namespace mss
 {
-
 ///
-/// @brief Perform the rcd_load operations - TARGET_TYPE_MCBIST specialization
-/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_MCBIST>
+/// @brief Perform the rcd_load operations - TARGET_TYPE_MCA specialization
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_MCA>
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
 template<>
-fapi2::ReturnCode rcd_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target )
+fapi2::ReturnCode rcd_load<TARGET_TYPE_MCA>( const fapi2::Target<TARGET_TYPE_MCA>& i_target )
 {
+    const auto& l_mcbist = mss::find_target<TARGET_TYPE_MCBIST>(i_target);
+
     // A vector of CCS instructions. We'll ask the targets to fill it, and then we'll execute it
     ccs::program<TARGET_TYPE_MCBIST> l_program;
 
@@ -69,28 +70,42 @@ fapi2::ReturnCode rcd_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_
 
     FAPI_TRY(mss::is_simulation(l_sim));
 
-    for ( const auto& p : mss::find_targets<TARGET_TYPE_MCA>(i_target) )
+    for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
     {
-        for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(p) )
+        // CKE needs to be LOW before running the RCW sequence
+        // So we use the power down entry command to achieve this
+        if(!l_sim)
         {
-            // CKE needs to be LOW before running the RCW sequence
-            // So we use the power down entry command to achieve this
-            if(!l_sim)
-            {
-                l_program.iv_instructions.push_back( ccs::pde_command<TARGET_TYPE_MCBIST>() );
-            }
-
-            FAPI_DBG("rcd load for %s", mss::c_str(d));
-            FAPI_TRY( perform_rcd_load(d, l_program.iv_instructions),
-                      "Failed perform_rcd_load() for %s", mss::c_str(d) );
-        }// dimms
-
-        // We have to configure the CCS engine to let it know which port these instructions are
-        // going out (or whether it's broadcast ...) so lets execute the instructions we presently
-        // have so that we kind of do this by port
-        FAPI_TRY( ccs::execute(i_target, l_program, p),
-                  "Failed to execute ccs for %s", mss::c_str(p) );
-    }// ports
+            l_program.iv_instructions.push_back( ccs::pde_command<TARGET_TYPE_MCBIST>() );
+        }
+
+        FAPI_DBG("rcd load for %s", mss::c_str(d));
+        FAPI_TRY( perform_rcd_load(d, l_program.iv_instructions),
+                  "Failed perform_rcd_load() for %s", mss::c_str(d) );
+    }// dimms
+
+    // We have to configure the CCS engine to let it know which port these instructions are
+    // going out (or whether it's broadcast ...) so lets execute the instructions we presently
+    // have so that we kind of do this by port
+    FAPI_TRY( ccs::execute(l_mcbist, l_program, i_target),
+              "Failed to execute ccs for %s", mss::c_str(i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Perform the rcd_load operations - TARGET_TYPE_MCBIST specialization
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_MCBIST>
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+template<>
+fapi2::ReturnCode rcd_load<TARGET_TYPE_MCBIST>( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target )
+{
+    for ( const auto& p : mss::find_targets<TARGET_TYPE_MCA>(i_target) )
+    {
+        FAPI_TRY( rcd_load(p) );
+    }
 
 fapi_try_exit:
     return fapi2::current_err;
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H b/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
index 641cb4d13..5f285f159 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
@@ -81,6 +81,7 @@ class mcTraits<fapi2::TARGET_TYPE_MCA>
             // Register definitions
             MBARPC0 = MCA_MBARPC0Q,
             MBASTR0 = MCA_MBASTR0Q,
+            FARB6Q = MCA_MBA_FARB6Q,
 
             SLOT0_VALID = MCS_PORT02_MCP0XLT0_SLOT0_VALID,
             SLOT0_ROW15_VALID = MCS_PORT02_MCP0XLT0_SLOT0_ROW15_VALID,
@@ -136,7 +137,8 @@ class mcTraits<fapi2::TARGET_TYPE_MCA>
             NM_CHANGE_AFTER_SYNC = MCA_MBA_FARB3Q_CFG_NM_CHANGE_AFTER_SYNC,
             CFG_MIN_MAX_DOMAINS = MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS,
             CFG_MIN_MAX_DOMAINS_LEN = MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS_LEN,
-
+            CKE_PUP_STATE = MCA_MBA_FARB6Q_CFG_CKE_PUP_STATE,
+            CKE_PUP_STATE_LEN = MCA_MBA_FARB6Q_CFG_CKE_PUP_STATE_LEN,
         };
 
 };
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.C b/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.C
index 211da063e..66086df35 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.C
@@ -722,6 +722,8 @@ fapi_try_exit:
 //    Defaults to the first address behind the target
 /// @param[in] i_end whether to end, and where
 ///   Defaults to stop after slave rank
+/// @param[in] i_end_address mcbist::address representing the address to end.
+//    Defaults to mcbist::address::LARGEST_ADDRESS
 /// @return FAPI2_RC_SUCCESS iff everything ok
 /// @note The function is asynchronous, and the caller should be looking for a done attention
 /// @note The address is often the port, dimm, rank but this is not enforced in the API.
@@ -730,12 +732,13 @@ template<>
 fapi2::ReturnCode sf_read( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target,
                            const stop_conditions& i_stop,
                            const mss::mcbist::address& i_address,
-                           const end_boundary i_end)
+                           const end_boundary i_end,
+                           const mss::mcbist::address& i_end_address )
 {
     FAPI_INF("superfast read - start for %s", mss::c_str(i_target));
 
     fapi2::ReturnCode l_rc;
-    constraints l_const(i_stop, speed::LUDICROUS, i_end, i_address);
+    constraints l_const(i_stop, speed::LUDICROUS, i_end, i_address, i_end_address);
     sf_read_operation<TARGET_TYPE_MCBIST> l_read_op(i_target, l_const, l_rc);
 
     FAPI_ASSERT( l_rc == FAPI2_RC_SUCCESS,
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.H b/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.H
index af5cd2c24..918342c6c 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/mcbist/memdiags.H
@@ -380,6 +380,8 @@ fapi2::ReturnCode broadcast_mode_start_address_check(const fapi2::Target<fapi2::
 //    Defaults to the first address behind the target
 /// @param[in] i_end whether to end, and where
 ///   Defaults to stop after slave rank
+/// @param[in] i_end_address mcbist::address representing the address to end.
+//    Defaults to mcbist::address::LARGEST_ADDRESS
 /// @return FAPI2_RC_SUCCESS iff everything ok
 /// @note The function is asynchronous, and the caller should be looking for a done attention
 /// @note The address is often the port, dimm, rank but this is not enforced in the API.
@@ -388,7 +390,8 @@ template< fapi2::TargetType T >
 fapi2::ReturnCode sf_read( const fapi2::Target<T>& i_target,
                            const stop_conditions& i_stop,
                            const mss::mcbist::address& i_address = mss::mcbist::address(),
-                           const end_boundary i_end = end_boundary::STOP_AFTER_SLAVE_RANK );
+                           const end_boundary i_end = end_boundary::STOP_AFTER_SLAVE_RANK,
+                           const mss::mcbist::address& i_end_address = mss::mcbist::address(mcbist::address::LARGEST_ADDRESS) );
 
 ///
 /// @brief Scrub - continuous scrub all memory behind the target
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H b/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
index e6027a580..9d3e8ded2 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
@@ -334,6 +334,16 @@ enum dimm_select
     DIMM_NONE        = 0b00,
 };
 
+// Possible values for power domains in MBARPC0Q
+enum min_max_domains : uint64_t
+{
+    MAXALL_MINALL = 0b000,
+    MAXALL_MIN1 = 0b001,
+    MAXALL_MIN0 = 0b010,
+    MAX1_MIN1 = 0b011,
+    MAX1_MIN0 = 0b100,
+};
+
 namespace mcbist
 {