NVDIMM RCD restore fix and zqcal timing fix

-The original recipe uses the rcd_load() from draminit to restore the RCD.
But that would not work because the powerdown mode is enabled on the RCD
which causes it to ignore the commands. Also, special consideration is
required since this is being done with data in the DRAMs.

-Reduces zqcal delay for NVDIMM

Change-Id: Ic504f185ef770857bb1efbc9e3e4f61656a4ecdc
CQ:SW458516
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/73507
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: ANDRE A. MARIN <aamarin@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/73519
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 320 insertions, 19 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H b/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
index 33469fe1b..6de8554b9 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -686,7 +686,8 @@ inline instruction_t<T> initial_cal_command(const uint64_t i_rp)
 ///
 template< fapi2::TargetType T, typename TT = ccsTraits<T> >
 inline instruction_t<T> zqcl_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
-                                      const uint64_t i_rank )
+                                      const uint64_t i_rank,
+                                      const uint16_t i_idle = 0 )
 {
     fapi2::buffer<uint64_t> l_boilerplate_arr0;
     fapi2::buffer<uint64_t> l_boilerplate_arr1;
@@ -705,6 +706,9 @@ inline instruction_t<T> zqcl_command( const fapi2::Target<fapi2::TARGET_TYPE_DIM
     // ADDR10/AP is high
     l_boilerplate_arr0.setBit<TT::ARR0_DDR_ADDRESS_10>();
 
+    // Insert idle
+    l_boilerplate_arr1.template insertFromRight<TT::ARR1_IDLES, TT::ARR1_IDLES_LEN>( i_idle );
+
     return instruction_t<T>(i_target, i_rank, l_boilerplate_arr0, l_boilerplate_arr1);
 }
 
@@ -870,6 +874,7 @@ inline instruction_t<T> rda_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM
 /// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
 /// @param[in] i_target the DIMM this instruction is headed for
 /// @param[in] i_rank the rank on this dimm
+/// @param[in] i_idle the idle time to the next command (default to 0)
 /// @return the precharge all banks command CCS instruction
 /// @note THIS IS DDR4 ONLY RIGHT NOW. We can (and possibly should) specialize this
 /// for the controller (Nimbus v Centaur) and then correct for DRAM generation (not included
@@ -877,7 +882,8 @@ inline instruction_t<T> rda_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM
 ///
 template< fapi2::TargetType T, typename TT = ccsTraits<T> >
 inline instruction_t<T> precharge_all_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
-        const uint64_t i_rank )
+        const uint64_t i_rank,
+        const uint16_t i_idle = 0 )
 {
     fapi2::buffer<uint64_t> l_boilerplate_arr0;
     fapi2::buffer<uint64_t> l_boilerplate_arr1;
@@ -896,6 +902,9 @@ inline instruction_t<T> precharge_all_command( const fapi2::Target<fapi2::TARGET
     // Setup ADDR10/AP high
     l_boilerplate_arr0.setBit<TT::ARR0_DDR_ADDRESS_10>();
 
+    // Insert idle
+    l_boilerplate_arr1.template insertFromRight<TT::ARR1_IDLES, TT::ARR1_IDLES_LEN>( i_idle );
+
     // From DDR4 Spec table 17:
     // All other bits from the command truth table or 'V', for valid (1 or 0)
 
@@ -908,12 +917,13 @@ inline instruction_t<T> precharge_all_command( const fapi2::Target<fapi2::TARGET
 /// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
 /// @param[in] i_target the DIMM this instruction is headed for
 /// @param[in] i_rank the rank on this dimm
+/// @param[in] i_idle the idle time to the next command (default to 0)
 /// @return the self-refresh entry command CCS instruction
 /// @note THIS IS FOR DDR4 NON-LRDIMM ONLY RIGHT NOW
 ///
 template< fapi2::TargetType T, typename TT = ccsTraits<T> >
 inline instruction_t<T> self_refresh_entry_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
-        const uint64_t i_rank )
+        const uint64_t i_rank, const uint16_t i_idle = 0 )
 {
     fapi2::buffer<uint64_t> l_boilerplate_arr0;
     fapi2::buffer<uint64_t> l_boilerplate_arr1;
@@ -929,6 +939,9 @@ inline instruction_t<T> self_refresh_entry_command( const fapi2::Target<fapi2::T
     .template clearBit<TT::ARR0_DDR_ADDRESS_15>()
     .template setBit<TT::ARR0_DDR_ADDRESS_14>();
 
+    // Insert idle
+    l_boilerplate_arr1.template insertFromRight<TT::ARR1_IDLES, TT::ARR1_IDLES_LEN>( i_idle );
+
     // From DDR4 Spec table 17:
     // All other bits from the command truth table are 'V', for valid (1 or 0)
 
@@ -941,13 +954,14 @@ inline instruction_t<T> self_refresh_entry_command( const fapi2::Target<fapi2::T
 /// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
 /// @param[in] i_target the DIMM this instruction is headed for
 /// @param[in] i_rank the rank on this dimm
+/// @param[in] i_idle the idle time to the next command (default to 0)
 /// @return the self-refresh exit command CCS instruction
 /// @note Using NOP in case SDRAM is in gear down mode and max power saving mode exit
 /// @note THIS IS FOR DDR4 NON-LRDIMM ONLY RIGHT NOW
 ///
 template< fapi2::TargetType T, typename TT = ccsTraits<T> >
 inline instruction_t<T> self_refresh_exit_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
-        const uint64_t i_rank )
+        const uint64_t i_rank, const uint16_t i_idle = 0 )
 {
     fapi2::buffer<uint64_t> l_boilerplate_arr0;
     fapi2::buffer<uint64_t> l_boilerplate_arr1;
@@ -963,6 +977,9 @@ inline instruction_t<T> self_refresh_exit_command( const fapi2::Target<fapi2::TA
     .template setBit<TT::ARR0_DDR_ADDRESS_15>()
     .template setBit<TT::ARR0_DDR_ADDRESS_14>();
 
+    // Insert idle
+    l_boilerplate_arr1.template insertFromRight<TT::ARR1_IDLES, TT::ARR1_IDLES_LEN>( i_idle );
+
     // From DDR4 Spec table 17:
     // All other bits from the command truth table are 'V', for valid (1 or 0)
 
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
index 735c5dab3..e564eafce 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
@@ -52,10 +52,12 @@
 
 #include <lib/mc/port.H>
 #include <lib/phy/dp16.H>
-#include <lib/dimm/rcd_load.H>
 #include <lib/dimm/mrs_load.H>
 #include <lib/dimm/ddr4/pda.H>
 #include <lib/dimm/ddr4/zqcal.H>
+#include <lib/dimm/ddr4/control_word_ddr4.H>
+#include <lib/workarounds/ccs_workarounds.H>
+#include <lib/eff_config/timing.H>
 
 using fapi2::TARGET_TYPE_MCBIST;
 using fapi2::TARGET_TYPE_MCA;
@@ -67,6 +69,7 @@ namespace mss
 namespace nvdimm
 {
 
+
 ///
 /// @brief Wrapper to read MAINT_ADDR_MODE_EN
 /// Specialization for TARGET_TYPE_MCA
@@ -171,8 +174,12 @@ fapi2::ReturnCode self_refresh_exit_helper( const fapi2::Target<fapi2::TARGET_TY
 {
 
     const auto& l_mcbist = mss::find_target<fapi2::TARGET_TYPE_MCBIST>(i_target);
-    fapi2::buffer<uint64_t> l_status, l_recr_buf, l_mcbfirq_buf, l_mcbfirmask_buf;
-    mss::states l_complete_mask = mss::states::LOW, l_wat_debug_attn_mask = mss::states::LOW;
+    fapi2::buffer<uint64_t> l_status;
+    fapi2::buffer<uint64_t> l_recr_buf;
+    fapi2::buffer<uint64_t> l_mcbfirq_buf;
+    fapi2::buffer<uint64_t> l_mcbfirmask_buf;
+    mss::states l_complete_mask = mss::states::LOW;
+    mss::states l_wat_debug_attn_mask = mss::states::LOW;
 
     // A small vector of addresses to poll during the polling loop
     const std::vector<mss::poll_probe<fapi2::TARGET_TYPE_MCBIST>> l_probes =
@@ -247,7 +254,6 @@ fapi_try_exit:
     return fapi2::current_err;
 }
 
-
 ///
 /// @brief Put target into self-refresh
 /// Specialization for TARGET_TYPE_MCA
@@ -380,6 +386,225 @@ fapi_try_exit:
 }
 
 ///
+/// @brief Disable powerdown mode in rc09
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
+        std::vector< ccs::instruction_t<TARGET_TYPE_MCBIST> >& io_inst)
+{
+    FAPI_INF("rc09_disable_powerdown %s", mss::c_str(i_target));
+
+    constexpr uint8_t POWER_DOWN_BIT = 4;
+    constexpr bool l_sim = false;
+    constexpr uint8_t FS0 = 0; // Function space 0
+    constexpr uint64_t CKE_HIGH = mss::ON;
+    fapi2::buffer<uint8_t> l_rc09_cw = 0;
+    std::vector<uint64_t> l_ranks;
+
+    FAPI_TRY(mss::eff_dimm_ddr4_rc09(i_target, l_rc09_cw));
+
+    // Clear power down enable bit.
+    l_rc09_cw.clearBit<POWER_DOWN_BIT>();
+
+    FAPI_TRY( mss::rank::ranks(i_target, l_ranks) );
+
+    // DES to ensure we exit powerdown properly
+    FAPI_DBG("deselect for %s", mss::c_str(i_target));
+    io_inst.push_back( ccs::des_command<TARGET_TYPE_MCBIST>() );
+
+    static const cw_data l_rc09_4bit_data( FS0, 9, l_rc09_cw, mss::tmrd() );
+
+    // Load RC09
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_HIGH),
+              "Failed to load 4-bit RC09 control word for %s",
+              mss::c_str(i_target));
+
+    // Hold the CKE high
+    mss::ccs::workarounds::hold_cke_high(io_inst);
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Load the rcd control words
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+/// @Note This is similar to rcd_load_ddr4() but with minor changes to support
+///       with NVDIMMs
+///
+fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
+                                   std::vector< ccs::instruction_t<TARGET_TYPE_MCBIST> >& io_inst)
+{
+    FAPI_INF("rcd_load_nvdimm %s", mss::c_str(i_target));
+
+    constexpr uint64_t CKE_LOW = mss::OFF;
+    constexpr bool l_sim = false;
+
+    // Per DDR4RCD02, tSTAB is us. We want this in cycles for the CCS.
+    const uint64_t tSTAB = mss::us_to_cycles(i_target, mss::tstab());
+    constexpr uint8_t FS0 = 0; // Function space 0
+
+    // RCD 4-bit data - integral represents rc#
+    static const std::vector< cw_data > l_rcd_4bit_data =
+    {
+        { FS0, 0,  eff_dimm_ddr4_rc00,    mss::tmrd()    },
+        { FS0, 1,  eff_dimm_ddr4_rc01,    mss::tmrd()    },
+        { FS0, 2,  eff_dimm_ddr4_rc02,    tSTAB          },
+        { FS0, 3,  eff_dimm_ddr4_rc03,    mss::tmrd_l()  },
+        { FS0, 4,  eff_dimm_ddr4_rc04,    mss::tmrd_l()  },
+        { FS0, 5,  eff_dimm_ddr4_rc05,    mss::tmrd_l()  },
+        // Note: the tMRC1 timing as it is larger for saftey's sake
+        // The concern is that if geardown mode is ever required in the future, we would need the longer timing
+        { FS0, 6,  eff_dimm_ddr4_rc06_07, mss::tmrc1()   },
+        { FS0, 8,  eff_dimm_ddr4_rc08,    mss::tmrd()    },
+        { FS0, 10, eff_dimm_ddr4_rc0a,    tSTAB          },
+        { FS0, 11, eff_dimm_ddr4_rc0b,    mss::tmrd_l()  },
+        { FS0, 12, eff_dimm_ddr4_rc0c,    mss::tmrd()    },
+        { FS0, 13, eff_dimm_ddr4_rc0d,    mss::tmrd_l2() },
+        { FS0, 14, eff_dimm_ddr4_rc0e,    mss::tmrd()    },
+        { FS0, 15, eff_dimm_ddr4_rc0f,    mss::tmrd_l2() },
+    };
+
+    // RCD 4-bit data - integral represents rc#
+    static const cw_data l_rc09_4bit_data( FS0, 9, eff_dimm_ddr4_rc09, mss::tmrd() );
+
+    // RCD 8-bit data - integral represents rc#
+    static const std::vector< cw_data > l_rcd_8bit_data =
+    {
+        { FS0, 1,  eff_dimm_ddr4_rc_1x, mss::tmrd()   },
+        { FS0, 2,  eff_dimm_ddr4_rc_2x, mss::tmrd()   },
+        { FS0, 3,  eff_dimm_ddr4_rc_3x, tSTAB         },
+        { FS0, 4,  eff_dimm_ddr4_rc_4x, mss::tmrd()   },
+        { FS0, 5,  eff_dimm_ddr4_rc_5x, mss::tmrd()   },
+        { FS0, 6,  eff_dimm_ddr4_rc_6x, mss::tmrd()   },
+        { FS0, 7,  eff_dimm_ddr4_rc_7x, mss::tmrd_l() },
+        { FS0, 8,  eff_dimm_ddr4_rc_8x, mss::tmrd()   },
+        { FS0, 9,  eff_dimm_ddr4_rc_9x, mss::tmrd()   },
+        { FS0, 10, eff_dimm_ddr4_rc_ax, mss::tmrd()   },
+        { FS0, 11, eff_dimm_ddr4_rc_bx, mss::tmrd_l() },
+    };
+
+    // Load 4-bit data
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rcd_4bit_data, l_sim, io_inst, CKE_LOW),
+              "Failed to load 4-bit control words for %s",
+              mss::c_str(i_target));
+
+    // Load 8-bit data
+    FAPI_TRY( control_word_engine<RCW_8BIT>(i_target, l_rcd_8bit_data, l_sim, io_inst, CKE_LOW),
+              "Failed to load 8-bit control words for %s",
+              mss::c_str(i_target));
+
+    // Load RC09 with CKE_LOW
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_LOW),
+              "Failed to load 4-bit RC09 control word for %s",
+              mss::c_str(i_target));
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Restore the rcd after restoring the nvdimm data
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_MCA>
+/// @return FAPI2_RC_SUCCESS if and only if ok
+/// @Note Restoring the RCD after NVDIMM restore requires a special procedure
+///       The procedure from draminit would actually fail to restore the CWs
+///
+fapi2::ReturnCode rcd_restore( const fapi2::Target<TARGET_TYPE_MCA>& i_target )
+{
+    const auto& l_mcbist = mss::find_target<TARGET_TYPE_MCBIST>(i_target);
+    std::vector<uint64_t> l_ranks;
+
+    // A vector of CCS instructions. We'll ask the targets to fill it, and then we'll execute it
+    ccs::program<TARGET_TYPE_MCBIST> l_program;
+
+    // Clear the initial delays. This will force the CCS engine to recompute the delay based on the
+    // instructions in the CCS instruction vector
+    l_program.iv_poll.iv_initial_delay = 0;
+    l_program.iv_poll.iv_initial_sim_delay = 0;
+
+    // We expect to come in with the port in STR. Before proceeding with
+    // restoring the RCD, power down needs to be disabled first on the RCD so
+    // the rest of the CWs can be restored with CKE low
+    for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
+    {
+        FAPI_DBG("rc09_disable_powerdown for %s", mss::c_str(d));
+        FAPI_TRY( rc09_disable_powerdown(d, l_program.iv_instructions),
+                  "Failed rc09_disable_powerdown() for %s", mss::c_str(d) );
+    }// dimms
+
+    // Exit STR first so CKE is back to high and rcd isn't ignoring us
+    FAPI_TRY( self_refresh_exit( i_target ) );
+
+    FAPI_TRY( ccs::execute(l_mcbist, l_program, i_target),
+              "Failed to execute ccs for %s", mss::c_str(i_target) );
+
+    // Now, drive CKE back to low via STR entry instead of pde (we have data in the drams!)
+    FAPI_TRY( self_refresh_entry( i_target ) );
+
+    l_program = ccs::program<TARGET_TYPE_MCBIST>(); //Reset the program
+
+    // Now, fill the program with instructions to program the RCD
+    for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
+    {
+        FAPI_DBG("rcd_load_nvdimm( for %s", mss::c_str(d));
+        FAPI_TRY( rcd_load_nvdimm(d, l_program.iv_instructions),
+                  "Failed rcd_load_nvdimm() for %s", mss::c_str(d) );
+    }// dimms
+
+    // Restore the rcd
+    FAPI_TRY( ccs::execute(l_mcbist, l_program, i_target),
+              "Failed to execute ccs for %s", mss::c_str(i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Execute post restore ZQCAL
+/// @param[in] i_target the target associated with this cal
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+/// @Note The original zqcal has delay that violates the refresh interval.
+///       Since we now have data in the DRAMs, the command will execute
+///       with delay following the spec.
+///
+fapi2::ReturnCode post_restore_zqcal( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target)
+{
+    const auto& l_mcbist = mss::find_target<TARGET_TYPE_MCBIST>(i_target);
+    std::vector<uint64_t> l_ranks;
+    uint8_t l_trp[MAX_DIMM_PER_PORT];
+    ccs::program<TARGET_TYPE_MCBIST> l_program;
+
+    // Get tRP
+    FAPI_TRY(mss::eff_dram_trp(mss::find_target<fapi2::TARGET_TYPE_MCS>(i_target), l_trp));
+
+    // Construct the program
+    for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
+    {
+        FAPI_TRY( mss::rank::ranks(d, l_ranks) );
+
+        for ( const auto r : l_ranks)
+        {
+            FAPI_DBG("precharge_all_command for %s", mss::c_str(d));
+            l_program.iv_instructions.push_back( ccs::precharge_all_command<TARGET_TYPE_MCBIST>(d, r, l_trp[0]) );
+            FAPI_DBG("zqcal_command for %s", mss::c_str(d));
+            l_program.iv_instructions.push_back( ccs::zqcl_command<TARGET_TYPE_MCBIST>(d, r, mss::tzqinit()) );
+        }
+    }// dimms
+
+    // execute ZQCAL instructions
+    FAPI_TRY( mss::ccs::execute(l_mcbist, l_program, i_target),
+              "Failed to execute ccs for ZQCAL %s", mss::c_str(i_target) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
 /// @brief Post restore transition to support restoring nvdimm to
 /// a functional state after the restoring the data from NAND flash
 /// to DRAM
@@ -401,8 +626,8 @@ fapi2::ReturnCode post_restore_transition( const fapi2::Target<fapi2::TARGET_TYP
         FAPI_TRY(change_maint_addr_mode_en(i_target, mss::states::LOW));
     }
 
-    // Subseqent restore on later nvdimms would go wonky if this goes before STR exit...
-    FAPI_TRY( mss::rcd_load( i_target ) );
+    // Restore the rcd
+    FAPI_TRY( rcd_restore( i_target ) );
 
     // Exit STR
     FAPI_TRY( self_refresh_exit( i_target ) );
@@ -411,13 +636,7 @@ fapi2::ReturnCode post_restore_transition( const fapi2::Target<fapi2::TARGET_TYP
     FAPI_TRY( mss::mrs_load( i_target ) );
 
     // Do ZQCAL
-    {
-        fapi2::buffer<uint32_t> l_cal_steps_enabled;
-        l_cal_steps_enabled.setBit<mss::DRAM_ZQCAL>();
-
-        FAPI_DBG("cal steps enabled: 0x%08x ", l_cal_steps_enabled);
-        FAPI_TRY( mss::setup_and_execute_zqcal(i_target, l_cal_steps_enabled), "Error in nvdimm setup_and_execute_zqcal()" );
-    }
+    FAPI_TRY( post_restore_zqcal(i_target) );
 
     // Latch the trained PDA vref values for each dimm under the port
     for (const auto& l_dimm : mss::find_targets<fapi2::TARGET_TYPE_DIMM>(i_target))
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
index 3c4f081d3..fe58fae31 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
@@ -36,6 +36,7 @@
 #include <fapi2.H>
 #include <generic/memory/lib/utils/find.H>
 #include <lib/shared/mss_const.H>
+#include <lib/ccs/ccs.H>
 
 namespace mss
 {
@@ -123,12 +124,50 @@ template< fapi2::TargetType T >
 fapi2::ReturnCode pda_vref_latch( const fapi2::Target<T>& i_target );
 
 ///
+/// @brief Disable powerdown mode in rc09
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+        std::vector< ccs::instruction_t<fapi2::TARGET_TYPE_MCBIST> >& io_inst);
+
+///
+/// @brief Load the rcd control words
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+/// @Note This is similar to rcd_load_ddr4() but with minor changes to support
+///       with NVDIMMs
+///
+fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                   std::vector< ccs::instruction_t<fapi2::TARGET_TYPE_MCBIST> >& io_inst);
+
+///
+/// @brief Restore the rcd after restoring the nvdimm data
+/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_MCA>
+/// @return FAPI2_RC_SUCCESS if and only if ok
+/// @Note Restoring the RCD after NVDIMM restore requires a special procedure
+///       The procedure from draminit would actually fail to restore the CWs
+///
+fapi2::ReturnCode rcd_restore( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target );
+
+///
+/// @brief Execute post restore ZQCAL
+/// @param[in] i_target the target associated with this cal
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+/// @Note The original zqcal has delay that violates the refresh interval.
+///       Since we now have data in the DRAMs, the command will be executed
+///       with delay following the spec.
+///
+fapi2::ReturnCode post_restore_zqcal( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target);
+
+///
 /// @brief Full post-restore transition for NVDIMM
 /// @tparam T the target type associated with this subroutine
 /// @param[in] i_target the target associated with this subroutine
 /// @return FAPI2_RC_SUCCESS iff setup was successful
 ///
-
 template< fapi2::TargetType T >
 fapi2::ReturnCode post_restore_transition( const fapi2::Target<T>& i_target );
 
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/eff_config/timing.H b/src/import/chips/p9/procedures/hwp/memory/lib/eff_config/timing.H
index 45c7ee19c..8b8215b96 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/eff_config/timing.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/eff_config/timing.H
@@ -377,6 +377,32 @@ constexpr uint8_t tadc()
 }
 
 ///
+/// @brief Self Refresh Entry delay
+/// @tparam T fapi2::TargetType of the target used to calculate cycles from ns
+/// @param[in] i_target the target used to get clocks
+/// @return max(5nCK,10ns) in clocks
+///
+template< fapi2::TargetType T >
+inline uint64_t tcksre( const fapi2::Target<T>& i_target )
+{
+    // Per DDR4 Full spec update (79-4A) - timing requirements
+    return mss::max_ck_ns<T>( i_target, 5, 10 );
+}
+
+///
+/// @brief Self Refresh Exit delay
+/// @tparam T fapi2::TargetType of the target used to calculate cycles from ns
+/// @param[in] i_target the target used to get clocks
+/// @return max(5nCK,10ns) in clocks
+///
+template< fapi2::TargetType T >
+inline uint64_t tcksrx( const fapi2::Target<T>& i_target )
+{
+    // Per DDR4 Full spec update (79-4A) - timing requirements
+    return mss::max_ck_ns<T>( i_target, 5, 10 );
+}
+
+///
 /// @brief DQS_t/DQS_n delay after write leveling mode is programmed
 /// @tparam T fapi2::TargetType of the target used to calculate cycles from ns
 /// @param[in] i_target the target used to get tMOD clocks