NVDIMM wr_vref workaround fix and add refreshes to ccs program (nvdimm only)

Change-Id: I0331284a284ac54ebd365abd96c67d4094b124b8
CQ:SW463048
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/77157
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/77198
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>

5 files changed, 251 insertions, 72 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H b/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
index d34ff6133..77f419d7a 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/mc/mc.H
@@ -123,6 +123,7 @@ class mcTraits<fapi2::TARGET_TYPE_MCA>
             EMERGENCY_M = MCA_MBA_FARB4Q_EMERGENCY_M,
             EMERGENCY_M_LEN = MCA_MBA_FARB4Q_EMERGENCY_M_LEN,
             CFG_STR_ENABLE = MCA_MBASTR0Q_CFG_STR_ENABLE,
+            CFG_STR_STATE = MCA_MBA_FARB6Q_CFG_STR_STATE,
 
             MIN_DOMAIN_REDUCTION_ENABLE = MCA_MBARPC0Q_CFG_MIN_DOMAIN_REDUCTION_ENABLE,
             MIN_MAX_DOMAINS_ENABLE = MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS_ENABLE,
@@ -189,6 +190,43 @@ enum
 
 namespace mc
 {
+///
+/// @brief Reads the contents of the FARB6Q
+/// @tparam T fapi2 Target Type - derived
+/// @tparam TT traits type defaults to mcTraits<T>
+/// @param[in] i_target the target on which to operate
+/// @param[out] o_data the register data
+/// @return fapi2::fapi2_rc_success if ok
+///
+template< fapi2::TargetType T, typename TT = mcTraits<T> >
+inline fapi2::ReturnCode read_farb6q( const fapi2::Target<T>& i_target, fapi2::buffer<uint64_t>& o_data )
+{
+    o_data = 0;
+
+    FAPI_TRY( mss::getScom(i_target, TT::FARB6Q, o_data ), "%s failed to read FARB6Q regiser", mss::c_str(i_target));
+    FAPI_DBG("%s FARB6Q has data 0x%016lx", mss::c_str(i_target), o_data);
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Writes the contents of the FARB6Q
+/// @tparam T fapi2 Target Type - derived
+/// @tparam TT traits type defaults to mcTraits<T>
+/// @param[in] i_target the target on which to operate
+/// @param[in] i_data the register data
+/// @return fapi2::fapi2_rc_success if ok
+///
+template< fapi2::TargetType T, typename TT = mcTraits<T> >
+inline fapi2::ReturnCode write_farb6q( const fapi2::Target<T>& i_target, const fapi2::buffer<uint64_t>& i_data )
+{
+    FAPI_TRY( mss::putScom(i_target, TT::FARB6Q, i_data ), "%s failed to write FARB6Q regiser", mss::c_str(i_target));
+    FAPI_DBG("%s FARB6Q has data 0x%016lx", mss::c_str(i_target), i_data);
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
 
 ///
 /// @brief Reads the contents of the MBARPC0
@@ -407,6 +445,20 @@ inline void get_enter_self_time_refresh_time( const fapi2::buffer<uint64_t>& i_d
 }
 
 ///
+/// @brief Gets the port self time refresh state
+/// @tparam T fapi2 Target Type - defaults to TARGET_TYPE_MCA
+/// @tparam TT traits type defaults to mcTraits<T>
+/// @param[in] i_data the value of the register
+/// @param[out] o_state the current STR state of the given port
+///
+template< fapi2::TargetType T = fapi2::TARGET_TYPE_MCA, typename TT = mcTraits<T> >
+inline void get_self_time_refresh_state( const fapi2::buffer<uint64_t>& i_data, mss::states& o_state )
+{
+    o_state = i_data.getBit<TT::CFG_STR_STATE>() ? mss::states::ON : mss::states::OFF;
+    FAPI_DBG("get_self_time_refresh_state to %d", o_state);
+}
+
+///
 /// @brief set the PWR CNTRL register
 /// @param[in] i_target the mca target
 /// @return fapi2::fapi2_rc_success if ok
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_training.C b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_training.C
index 892a72dd5..7501a0884 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_training.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_training.C
@@ -767,7 +767,7 @@ fapi2::ReturnCode write_ctr::post_workaround( const fapi2::Target<fapi2::TARGET_
     // As such, let's run the nvdimm with rank median vref value so later on we can just load
     // the vref in 1 ccs sequence.
     if ((l_hybrid[0] ==  fapi2::ENUM_ATTR_EFF_HYBRID_IS_HYBRID) &&
-        (l_hybrid_type[0] == fapi2::ENUM_ATTR_EFF_HYBRID_MEMORY_TYPE_NVDIMM) && !iv_wr_vref)
+        (l_hybrid_type[0] == fapi2::ENUM_ATTR_EFF_HYBRID_MEMORY_TYPE_NVDIMM) && iv_wr_vref)
     {
         FAPI_TRY(mss::workarounds::wr_vref::nvdimm_workaround(i_target, i_rp, i_abort_on_error));
 
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C
index 52572064f..9cc041ed6 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C
@@ -43,6 +43,7 @@
 #include <lib/eff_config/timing.H>
 #include <generic/memory/lib/ccs/ccs.H>
 #include <lib/mc/port.H>
+#include <lib/mc/mc.H>
 
 namespace mss
 {
@@ -221,6 +222,66 @@ namespace nvdimm
 {
 
 ///
+/// @brief add_refreshes() helper
+/// @param[in] i_target The MCA target where the program will be executed on
+/// @param[in,out] i_program the MCBIST ccs program to append the refreshes
+/// @return FAPI2_RC_SUCCESS iff success
+///
+fapi2::ReturnCode add_refreshes_helper(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                       mss::ccs::program& io_program)
+{
+    typedef ccsTraits<mss::mc_type::NIMBUS> TT;
+
+    uint16_t l_trefi = 0;
+
+    // 3 refreshes because the maximum number of instructions we can have is 28 for
+    // dual-rank NVDIMM (MRS)
+    constexpr size_t l_num_refreshes = 3;
+    constexpr uint64_t CS_N_ACTIVE = 0b00;
+
+    //get tREFI
+    FAPI_TRY(mss::eff_dram_trefi(i_target, l_trefi));
+
+    //load the refreshes into the program
+    for (size_t i = 0; i < l_num_refreshes; i++)
+    {
+        // We want to make sure the refresh hit all the ranks, so let's get the refresh command and change the chip select manually later
+        mss::ccs::instruction_t l_inst = mss::ccs::refresh_command(0, l_trefi);
+        l_inst.arr0.insertFromRight<TT::ARR0_DDR_CSN_0_1, TT::ARR0_DDR_CSN_0_1_LEN>(CS_N_ACTIVE);
+        l_inst.arr0.insertFromRight<TT::ARR0_DDR_CSN_2_3, TT::ARR0_DDR_CSN_2_3_LEN>(CS_N_ACTIVE);
+        l_inst.iv_update_rank = false;
+        io_program.iv_instructions.push_back(l_inst);
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Adds refreshes at the beginning and end of the program
+/// @param[in] i_target The MCA target where the program will be executed on
+/// @param[in,out] io_program the MCBIST ccs program to add the refreshes
+/// @return FAPI2_RC_SUCCESS iff success
+///
+fapi2::ReturnCode add_refreshes(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                mss::ccs::program& io_program)
+{
+    mss::ccs::program l_program;
+
+    // Add the refreshes at the beginning
+    FAPI_TRY(add_refreshes_helper(i_target, l_program));
+
+    // Append the instructions from io_program
+    l_program.iv_instructions.insert(l_program.iv_instructions.end(), io_program.iv_instructions.begin(),
+                                     io_program.iv_instructions.end());
+
+    io_program = l_program;
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
 /// @brief Execute the contents of the CCS array with ccs_addr_mux_sel control
 /// @param[in] i_target The MCBIST containing the array
 /// @param[in] i_program the MCBIST ccs program - to get the polling parameters
@@ -291,96 +352,121 @@ fapi2::ReturnCode execute( const fapi2::Target<fapi2::TARGET_TYPE_MCBIST>& i_tar
     constexpr size_t CCS_INSTRUCTION_DEPTH = 32 - 1;
     constexpr uint64_t CCS_ARR0_ZERO = MCBIST_CCS_INST_ARR0_00;
     constexpr uint64_t CCS_ARR1_ZERO = MCBIST_CCS_INST_ARR1_00;
+    mss::states l_str_state = mss::states::OFF;
+    fapi2::buffer<uint64_t> l_farb6q;
 
     mss::ccs::instruction_t l_des = ccs::des_command();
 
     FAPI_INF("loading ccs instructions (%d) for %s", i_program.iv_instructions.size(), mss::c_str(i_target));
 
-    auto l_inst_iter = i_program.iv_instructions.begin();
-
-    // Stop the CCS engine just for giggles - it might be running ...
-    FAPI_TRY( mss::ccs::start_stop(i_target, mss::states::STOP), "Error in ccs::execute" );
+    //Check if we are in str. If we are not, throw some refreshes into the program
+    FAPI_TRY(mss::mc::read_farb6q(i_port, l_farb6q));
+    mss::mc::get_self_time_refresh_state(l_farb6q, l_str_state);
 
-    FAPI_ASSERT( mss::poll(i_target, TT::STATQ_REG, poll_parameters(),
-                           [](const size_t poll_remaining, const fapi2::buffer<uint64_t>& stat_reg) -> bool
+    if (l_str_state == mss::states::OFF)
     {
-        FAPI_INF("ccs statq (stop) 0x%llx, remaining: %d", stat_reg, poll_remaining);
-        return stat_reg.getBit<TT::CCS_IN_PROGRESS>() != 1;
-    }),
-    fapi2::MSS_CCS_HUNG_TRYING_TO_STOP().set_MCBIST_TARGET(i_target) );
+        // Since we are executing the CCS program with data in the DRAMs, we need to be congnizant
+        // about the refreshes. Refresh from mc is fenced off when CCS has the bus, and by the time
+        // the control is given back to the mc, we would have violated 8*trefi refresh window.
+        // As such, let's start off each program with couple of refreshes so we don't violate the
+        // rolling 8*trefi window (verified on logic analyzer)
+        FAPI_TRY(add_refreshes(i_port, i_program));
+    }
 
-    while (l_inst_iter != i_program.iv_instructions.end())
     {
-        size_t l_inst_count = 0;
+        auto l_inst_iter = i_program.iv_instructions.begin();
 
-        uint64_t l_total_delay = 0;
-        uint64_t l_delay = 0;
-        uint64_t l_repeat = 0;
-        uint8_t l_current_cke = 0;
+        std::vector<rank_configuration> l_rank_configs;
+        FAPI_TRY(get_rank_config(i_target, l_rank_configs));
 
-        // Shove the instructions into the CCS engine, in 32 instruction chunks, and execute them
-        for (; l_inst_iter != i_program.iv_instructions.end()
-             && l_inst_count < CCS_INSTRUCTION_DEPTH; ++l_inst_count, ++l_inst_iter)
+        // Stop the CCS engine just for giggles - it might be running ...
+        FAPI_TRY( mss::ccs::start_stop(i_target, mss::states::STOP), "Error in ccs::execute" );
+
+        FAPI_ASSERT( mss::poll(i_target, TT::STATQ_REG, poll_parameters(),
+                               [](const size_t poll_remaining, const fapi2::buffer<uint64_t>& stat_reg) -> bool
         {
-            l_inst_iter->arr0.extractToRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
+            FAPI_INF("ccs statq (stop) 0x%llx, remaining: %d", stat_reg, poll_remaining);
+            return stat_reg.getBit<TT::CCS_IN_PROGRESS>() != 1;
+        }),
+        fapi2::MSS_CCS_HUNG_TRYING_TO_STOP().set_MCBIST_TARGET(i_target) );
 
-            // Make sure this instruction leads to the next. Notice this limits this mechanism to pretty
-            // simple (straight line) CCS programs. Anything with a loop or such will need another mechanism.
-            l_inst_iter->arr1.insertFromRight<MCBIST_CCS_INST_ARR1_00_GOTO_CMD,
-                        MCBIST_CCS_INST_ARR1_00_GOTO_CMD_LEN>(l_inst_count + 1);
-            FAPI_TRY( mss::putScom(i_target, CCS_ARR0_ZERO + l_inst_count, l_inst_iter->arr0), "Error in ccs::execute" );
-            FAPI_TRY( mss::putScom(i_target, CCS_ARR1_ZERO + l_inst_count, l_inst_iter->arr1), "Error in ccs::execute" );
+        while (l_inst_iter != i_program.iv_instructions.end())
+        {
+            size_t l_inst_count = 0;
 
-            // arr1 contains a specification of the delay and repeat after this instruction, as well
-            // as a repeat. Total up the delays as we go so we know how long to wait before polling
-            // the CCS engine for completion
-            l_inst_iter->arr1.extractToRight<MCBIST_CCS_INST_ARR1_00_IDLES, MCBIST_CCS_INST_ARR1_00_IDLES_LEN>(l_delay);
-            l_inst_iter->arr1.extractToRight<MCBIST_CCS_INST_ARR1_00_REPEAT_CMD_CNT,
-                        MCBIST_CCS_INST_ARR1_00_REPEAT_CMD_CNT>(l_repeat);
+            uint64_t l_total_delay = 0;
+            uint64_t l_delay = 0;
+            uint64_t l_repeat = 0;
+            uint8_t l_current_cke = 0;
+            const auto l_port_index = mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_port);
 
-            l_total_delay += l_delay * (l_repeat + 1);
+            // Shove the instructions into the CCS engine, in 32 instruction chunks, and execute them
+            for (; l_inst_iter != i_program.iv_instructions.end()
+                 && l_inst_count < CCS_INSTRUCTION_DEPTH; ++l_inst_count, ++l_inst_iter)
+            {
+                // First, update the current instruction's chip selects for the current port
+                FAPI_TRY(l_inst_iter->configure_rank(i_port, l_rank_configs[l_port_index]), "Error in rank config");
 
-            FAPI_INF("css inst %d: 0x%016lX 0x%016lX (0x%lx, 0x%lx) delay: 0x%x (0x%x) %s",
-                     l_inst_count, l_inst_iter->arr0, l_inst_iter->arr1,
-                     CCS_ARR0_ZERO + l_inst_count, CCS_ARR1_ZERO + l_inst_count,
-                     l_delay, l_total_delay, mss::c_str(i_target));
-        }
+                l_inst_iter->arr0.extractToRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
 
-        // Check our program for any delays. If there isn't a iv_initial_delay configured, then
-        // we use the delay we just summed from the instructions.
-        if (i_program.iv_poll.iv_initial_delay == 0)
-        {
-            i_program.iv_poll.iv_initial_delay = cycles_to_ns(i_target, l_total_delay);
-        }
+                // Make sure this instruction leads to the next. Notice this limits this mechanism to pretty
+                // simple (straight line) CCS programs. Anything with a loop or such will need another mechanism.
+                l_inst_iter->arr1.insertFromRight<MCBIST_CCS_INST_ARR1_00_GOTO_CMD,
+                            MCBIST_CCS_INST_ARR1_00_GOTO_CMD_LEN>(l_inst_count + 1);
+                FAPI_TRY( mss::putScom(i_target, CCS_ARR0_ZERO + l_inst_count, l_inst_iter->arr0), "Error in ccs::execute" );
+                FAPI_TRY( mss::putScom(i_target, CCS_ARR1_ZERO + l_inst_count, l_inst_iter->arr1), "Error in ccs::execute" );
+
+                // arr1 contains a specification of the delay and repeat after this instruction, as well
+                // as a repeat. Total up the delays as we go so we know how long to wait before polling
+                // the CCS engine for completion
+                l_inst_iter->arr1.extractToRight<MCBIST_CCS_INST_ARR1_00_IDLES, MCBIST_CCS_INST_ARR1_00_IDLES_LEN>(l_delay);
+                l_inst_iter->arr1.extractToRight<MCBIST_CCS_INST_ARR1_00_REPEAT_CMD_CNT,
+                            MCBIST_CCS_INST_ARR1_00_REPEAT_CMD_CNT>(l_repeat);
+
+                l_total_delay += l_delay * (l_repeat + 1);
+
+                FAPI_INF("css inst %d: 0x%016lX 0x%016lX (0x%lx, 0x%lx) delay: 0x%x (0x%x) %s",
+                         l_inst_count, l_inst_iter->arr0, l_inst_iter->arr1,
+                         CCS_ARR0_ZERO + l_inst_count, CCS_ARR1_ZERO + l_inst_count,
+                         l_delay, l_total_delay, mss::c_str(i_target));
+            }
 
-        if (i_program.iv_poll.iv_initial_sim_delay == 0)
-        {
-            i_program.iv_poll.iv_initial_sim_delay = cycles_to_simcycles(l_total_delay);
-        }
+            // Check our program for any delays. If there isn't a iv_initial_delay configured, then
+            // we use the delay we just summed from the instructions.
+            if (i_program.iv_poll.iv_initial_delay == 0)
+            {
+                i_program.iv_poll.iv_initial_delay = cycles_to_ns(i_target, l_total_delay);
+            }
 
-        FAPI_INF("executing ccs instructions (%d:%d, %d) for %s",
-                 i_program.iv_instructions.size(), l_inst_count, i_program.iv_poll.iv_initial_delay, mss::c_str(i_target));
-
-        // Deselect
-        l_des.arr0.insertFromRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
-
-        // Insert a DES as our last instruction. DES is idle state anyway and having this
-        // here as an instruction forces the CCS engine to wait the delay specified in
-        // the last instruction in this array (which it otherwise doesn't do.)
-        l_des.arr1.setBit<MCBIST_CCS_INST_ARR1_00_END>();
-        FAPI_TRY( mss::putScom(i_target, CCS_ARR0_ZERO + l_inst_count, l_des.arr0), "Error in ccs::execute" );
-        FAPI_TRY( mss::putScom(i_target, CCS_ARR1_ZERO + l_inst_count, l_des.arr1), "Error in ccs::execute" );
-
-        FAPI_INF("css inst %d fixup: 0x%016lX 0x%016lX (0x%lx, 0x%lx) %s",
-                 l_inst_count, l_des.arr0, l_des.arr1,
-                 CCS_ARR0_ZERO + l_inst_count, CCS_ARR1_ZERO + l_inst_count, mss::c_str(i_target));
-
-        // Kick off the CCS engine - per port. No broadcast mode for CCS (per Shelton 9/23/15)
-        FAPI_INF("executing CCS array for port %d (%s)", mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_port),
-                 mss::c_str(i_port));
-        FAPI_TRY( mss::ccs::select_ports( i_target, mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_port)),
-                  "Error in ccs execute" );
-        FAPI_TRY( execute_inst_array(i_target, i_program, i_port), "Error in ccs execute" );
+            if (i_program.iv_poll.iv_initial_sim_delay == 0)
+            {
+                i_program.iv_poll.iv_initial_sim_delay = cycles_to_simcycles(l_total_delay);
+            }
+
+            FAPI_INF("executing ccs instructions (%d:%d, %d) for %s",
+                     i_program.iv_instructions.size(), l_inst_count, i_program.iv_poll.iv_initial_delay, mss::c_str(i_target));
+
+            // Deselect
+            l_des.arr0.insertFromRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
+
+            // Insert a DES as our last instruction. DES is idle state anyway and having this
+            // here as an instruction forces the CCS engine to wait the delay specified in
+            // the last instruction in this array (which it otherwise doesn't do.)
+            l_des.arr1.setBit<MCBIST_CCS_INST_ARR1_00_END>();
+            FAPI_TRY( mss::putScom(i_target, CCS_ARR0_ZERO + l_inst_count, l_des.arr0), "Error in ccs::execute" );
+            FAPI_TRY( mss::putScom(i_target, CCS_ARR1_ZERO + l_inst_count, l_des.arr1), "Error in ccs::execute" );
+
+            FAPI_INF("css inst %d fixup: 0x%016lX 0x%016lX (0x%lx, 0x%lx) %s",
+                     l_inst_count, l_des.arr0, l_des.arr1,
+                     CCS_ARR0_ZERO + l_inst_count, CCS_ARR1_ZERO + l_inst_count, mss::c_str(i_target));
+
+            // Kick off the CCS engine - per port. No broadcast mode for CCS (per Shelton 9/23/15)
+            FAPI_INF("executing CCS array for port %d (%s)", mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_port),
+                     mss::c_str(i_port));
+            FAPI_TRY( mss::ccs::select_ports( i_target, mss::relative_pos<fapi2::TARGET_TYPE_MCBIST>(i_port)),
+                      "Error in ccs execute" );
+            FAPI_TRY( execute_inst_array(i_target, i_program, i_port), "Error in ccs execute" );
+        }
     }
 
 fapi_try_exit:
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H
index 6cf5f7900..8a4773321 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H
@@ -164,6 +164,24 @@ namespace nvdimm
 {
 
 ///
+/// @brief add_refreshes() helper
+/// @param[in] i_target The MCA target where the program will be executed on
+/// @param[in,out] i_program the MCBIST ccs program to append the refreshes
+/// @return FAPI2_RC_SUCCESS iff success
+///
+fapi2::ReturnCode add_refreshes_helper(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                       mss::ccs::program& io_program);
+
+///
+/// @brief Adds refreshes at the beginning and end of the program
+/// @param[in] i_target The MCA target where the program will be executed on
+/// @param[in,out] io_program the MCBIST ccs program to add the refreshes
+/// @return FAPI2_RC_SUCCESS iff success
+///
+fapi2::ReturnCode add_refreshes(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+                                mss::ccs::program& io_program);
+
+///
 /// @brief Execute a set of CCS instructions
 /// @param[in] i_target the target to effect
 /// @param[in] i_program the vector of instructions
diff --git a/src/import/generic/memory/lib/ccs/ccs.H b/src/import/generic/memory/lib/ccs/ccs.H
index b72ec64ae..aafb0826e 100644
--- a/src/import/generic/memory/lib/ccs/ccs.H
+++ b/src/import/generic/memory/lib/ccs/ccs.H
@@ -883,6 +883,29 @@ inline instruction_t self_refresh_exit_command( const uint64_t i_rank, const uin
     return instruction_t(i_rank, l_boilerplate_arr0, l_boilerplate_arr1);
 }
 
+///
+/// @brief Setup refresh command instruction
+/// @tparam T the target type of the chiplet which executes the CCS instruction
+/// @tparam TT the CCS traits of the chiplet which executes the CCS instruction
+/// @param[in] i_target the DIMM this instruction is headed for
+/// @param[in] i_rank the rank on this dimm
+/// @param[in] i_idle the idle time to the next command (default to 0)
+/// @return the self-refresh entry command CCS instruction
+/// @note THIS IS FOR DDR4 NON-LRDIMM ONLY RIGHT NOW
+///
+inline instruction_t refresh_command( const uint64_t i_rank, const uint16_t i_idle = 0 )
+{
+    using TT = ccsTraits<DEFAULT_MC_TYPE>;
+
+    // Refresh is self-refresh entry with CKE high
+    auto l_refresh_template = self_refresh_entry_command(i_rank, i_idle);
+
+    // CKE is high
+    l_refresh_template.arr0.template insertFromRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(CKE_HIGH);
+
+    return l_refresh_template;
+}
+
 //
 // These functions are a little sugar to keep callers from doing the traits-dance to get the
 // appropriate bit field