Re-order NVDIMM MRS restore sequence to comply with Jedec spec

Also does the following:
1. Remove STR from RCD restore sequence (non-issue for IDT RCD)
2. MCFGP_VALID api for HB to consume during MPIPL

Change-Id: I7f00d96d286e1568bcf2580afe9cf11111110b5e
CQ:SW470208
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80470
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Dean Sanner <dsanner@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80584
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M Crowell <dcrowell@us.ibm.com>

5 files changed, 130 insertions, 100 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C
index a3b88797b..3ac509b6c 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C
@@ -146,28 +146,82 @@ fapi2::ReturnCode mrs_load( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
     const size_t DOUBLE_TMRD = 2 * mss::tmrd();
     const size_t DOUBLE_TMOD = 2 * mss::tmod(i_target);
 
-    static const std::vector< mrs_data<TARGET_TYPE_MCBIST> > MRS_DATA =
+    // tDLLK to wait for DLL Reset
+    uint64_t tDLLK = 0;
+    FAPI_TRY( mss::tdllk(i_target, tDLLK) );
+
     {
-        // JEDEC ordering of MRS per DDR4 power on sequence
-        {  3, mrs03, mrs03_decode, DOUBLE_TMRD  },
-        {  6, mrs06, mrs06_decode, DOUBLE_TMRD  },
-        {  5, mrs05, mrs05_decode, DOUBLE_TMRD  },
-        {  4, mrs04, mrs04_decode, DOUBLE_TMRD  },
-        {  2, mrs02, mrs02_decode, DOUBLE_TMRD  },
-        {  1, mrs01, mrs01_decode, DOUBLE_TMRD  },
-        // We need to wait tmod before zqcl, a non-mrs command
-        {  0, mrs00, mrs00_decode, DOUBLE_TMOD },
-    };
-
-    std::vector< uint64_t > l_ranks;
-    FAPI_TRY( mss::rank::ranks(i_target, l_ranks) );
-
-    // Load MRS
-    for (const auto& d : MRS_DATA)
+        const std::vector< mrs_data<TARGET_TYPE_MCBIST> > MRS_DATA =
+        {
+            // JEDEC ordering of MRS per DDR4 power on sequence
+            {  3, mrs03, mrs03_decode, DOUBLE_TMRD  },
+            {  6, mrs06, mrs06_decode, DOUBLE_TMRD  },
+            {  5, mrs05, mrs05_decode, DOUBLE_TMRD  },
+            {  4, mrs04, mrs04_decode, DOUBLE_TMRD  },
+            {  2, mrs02, mrs02_decode, DOUBLE_TMRD  },
+            {  1, mrs01, mrs01_decode, DOUBLE_TMRD  },
+            // We need to wait tmod before zqcl, a non-mrs command
+            // Adding Per Glancy's request, to ensure DLL locking time
+            {  0, mrs00, mrs00_decode, DOUBLE_TMOD + tDLLK },
+        };
+
+        std::vector< uint64_t > l_ranks;
+        FAPI_TRY( mss::rank::ranks(i_target, l_ranks) );
+
+        // Load MRS
+        for (const auto& d : MRS_DATA)
+        {
+            for (const auto& r : l_ranks)
+            {
+                FAPI_TRY( mrs_engine(i_target, d, r, io_inst) );
+            }
+        }
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Perform the mrs_load DDR4 operations for nvdimm restore - TARGET_TYPE_DIMM specialization
+/// @param[in] i_target a fapi2::Target<fapi2::TARGET_TYPE_DIMM>
+/// @param[in] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode mrs_load_nvdimm( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
+                                   std::vector< ccs::instruction_t >& io_inst)
+{
+    FAPI_INF("ddr4::mrs_load_nvdimm %s", mss::c_str(i_target));
+
+    // tDLLK to wait for DLL Reset
+    uint64_t tDLLK = 0;
+    FAPI_TRY( mss::tdllk(i_target, tDLLK) );
+
     {
-        for (const auto& r : l_ranks)
+        const std::vector< mrs_data<TARGET_TYPE_MCBIST> > MRS_DATA =
+        {
+            // JEDEC ordering of MRS per DDR4 NVDIMM restore sequence
+            // Need to perform DLL off to on procedure (mrs01&mrs00)
+            // before all the other MRS's
+            {  1, mrs01, mrs01_decode, mss::tmrd()  },
+            {  0, mrs00, mrs00_decode, mss::tmod(i_target) + tDLLK },
+            {  3, mrs03, mrs03_decode, mss::tmrd()  },
+            {  6, mrs06, mrs06_decode, mss::tmrd()  },
+            {  5, mrs05, mrs05_decode, mss::tmrd()  },
+            {  4, mrs04, mrs04_decode, mss::tmrd()  },
+            {  2, mrs02, mrs02_decode, mss::tmrd()  },
+        };
+
+        std::vector< uint64_t > l_ranks;
+        FAPI_TRY( mss::rank::ranks(i_target, l_ranks) );
+
+        // Load MRS
+        for (const auto& d : MRS_DATA)
         {
-            FAPI_TRY( mrs_engine(i_target, d, r, io_inst) );
+            for (const auto& r : l_ranks)
+            {
+                FAPI_TRY( mrs_engine(i_target, d, r, io_inst) );
+            }
         }
     }
 
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H
index f9d0ac062..5e6aa5c35 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H
@@ -1511,6 +1511,15 @@ fapi2::ReturnCode mrs_load( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_targ
                             std::vector< ccs::instruction_t >& io_inst);
 
 ///
+/// @brief Perform the mrs_load DDR4 operations for nvdimm restore - TARGET_TYPE_DIMM specialization
+/// @param[in] i_target a fapi2::Target<fapi2::TARGET_TYPE_DIMM>
+/// @param[in] io_inst a vector of CCS instructions we should add to
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+fapi2::ReturnCode mrs_load_nvdimm( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                   std::vector< ccs::instruction_t >& io_inst);
+
+///
 /// @brief Sets WR LVL mode
 /// @param[in] i_target a DIMM target
 /// @param[in] i_mode setting for WR LVL mode
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
index 9178379dd..0d517ffbc 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C
@@ -328,49 +328,6 @@ fapi_try_exit:
 }
 
 ///
-/// @brief Disable powerdown mode in rc09
-/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
-/// @param[in,out] io_inst a vector of CCS instructions we should add to
-/// @return FAPI2_RC_SUCCESS if and only if ok
-///
-fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
-        std::vector< ccs::instruction_t >& io_inst)
-{
-    FAPI_INF("rc09_disable_powerdown %s", mss::c_str(i_target));
-
-    constexpr uint8_t POWER_DOWN_BIT = 4;
-    constexpr bool l_sim = false;
-    constexpr uint8_t FS0 = 0; // Function space 0
-    constexpr uint64_t CKE_HIGH = mss::ON;
-    fapi2::buffer<uint8_t> l_rc09_cw = 0;
-    std::vector<uint64_t> l_ranks;
-
-    FAPI_TRY(mss::eff_dimm_ddr4_rc09(i_target, l_rc09_cw));
-
-    // Clear power down enable bit.
-    l_rc09_cw.clearBit<POWER_DOWN_BIT>();
-
-    FAPI_TRY( mss::rank::ranks(i_target, l_ranks) );
-
-    // DES to ensure we exit powerdown properly
-    FAPI_DBG("deselect for %s", mss::c_str(i_target));
-    io_inst.push_back( ccs::des_command() );
-
-    static const cw_data l_rc09_4bit_data( FS0, 9, l_rc09_cw, mss::tmrd() );
-
-    // Load RC09
-    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_HIGH),
-              "Failed to load 4-bit RC09 control word for %s",
-              mss::c_str(i_target));
-
-    // Hold the CKE high
-    mss::ccs::workarounds::hold_cke_high(io_inst);
-
-fapi_try_exit:
-    return fapi2::current_err;
-}
-
-///
 /// @brief Load the rcd control words
 /// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
 /// @param[in,out] io_inst a vector of CCS instructions we should add to
@@ -383,7 +340,7 @@ fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target<TARGET_TYPE_DIMM>& i_targ
 {
     FAPI_INF("rcd_load_nvdimm %s", mss::c_str(i_target));
 
-    constexpr uint64_t CKE_LOW = mss::OFF;
+    constexpr uint64_t CKE_HIGH = mss::ON;
     constexpr bool l_sim = false;
 
     // Per DDR4RCD02, tSTAB is us. We want this in cycles for the CCS.
@@ -431,17 +388,19 @@ fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target<TARGET_TYPE_DIMM>& i_targ
     };
 
     // Load 4-bit data
-    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rcd_4bit_data, l_sim, io_inst, CKE_LOW),
+    // Keeping the CKE high as this will be done not in powerdown/STR mode. Not affected for
+    // the RCD supplier on nvdimm
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rcd_4bit_data, l_sim, io_inst, CKE_HIGH),
               "Failed to load 4-bit control words for %s",
               mss::c_str(i_target));
 
     // Load 8-bit data
-    FAPI_TRY( control_word_engine<RCW_8BIT>(i_target, l_rcd_8bit_data, l_sim, io_inst, CKE_LOW),
+    FAPI_TRY( control_word_engine<RCW_8BIT>(i_target, l_rcd_8bit_data, l_sim, io_inst, CKE_HIGH),
               "Failed to load 8-bit control words for %s",
               mss::c_str(i_target));
 
     // Load RC09 with CKE_LOW
-    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_LOW),
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_HIGH),
               "Failed to load 4-bit RC09 control word for %s",
               mss::c_str(i_target));
 
@@ -469,27 +428,6 @@ fapi2::ReturnCode rcd_restore( const fapi2::Target<TARGET_TYPE_MCA>& i_target )
     l_program.iv_poll.iv_initial_delay = 0;
     l_program.iv_poll.iv_initial_sim_delay = 0;
 
-    // We expect to come in with the port in STR. Before proceeding with
-    // restoring the RCD, power down needs to be disabled first on the RCD so
-    // the rest of the CWs can be restored with CKE low
-    for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
-    {
-        FAPI_DBG("rc09_disable_powerdown for %s", mss::c_str(d));
-        FAPI_TRY( rc09_disable_powerdown(d, l_program.iv_instructions),
-                  "Failed rc09_disable_powerdown() for %s", mss::c_str(d) );
-    }// dimms
-
-    // Exit STR first so CKE is back to high and rcd isn't ignoring us
-    FAPI_TRY( self_refresh_exit( i_target ) );
-
-    FAPI_TRY( mss::ccs::workarounds::nvdimm::execute(l_mcbist, l_program, i_target),
-              "Failed to execute ccs for %s", mss::c_str(i_target) );
-
-    // Now, drive CKE back to low via STR entry instead of pde (we have data in the drams!)
-    FAPI_TRY( self_refresh_entry( i_target ) );
-
-    l_program = ccs::program(); //Reset the program
-
     // Now, fill the program with instructions to program the RCD
     for ( const auto& d : mss::find_targets<TARGET_TYPE_DIMM>(i_target) )
     {
@@ -614,12 +552,12 @@ fapi2::ReturnCode post_restore_transition( const fapi2::Target<fapi2::TARGET_TYP
     FAPI_TRY(get_refresh_overrun_mask(i_target, l_refresh_overrun_mask));
     FAPI_TRY(change_refresh_overrun_mask(i_target, mss::states::ON));
 
-    // Restore the rcd
-    FAPI_TRY( rcd_restore( i_target ) );
-
     // Exit STR
     FAPI_TRY( self_refresh_exit( i_target ) );
 
+    // Restore the rcd
+    FAPI_TRY( rcd_restore( i_target ) );
+
     // Load the MRS
     FAPI_TRY( mss::mrs_load( i_target, NVDIMM_WORKAROUND ) );
 
@@ -641,6 +579,26 @@ fapi_try_exit:
 }
 
 ///
+/// @brief Helper to change the BAR valid state. Consumed by hostboot
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+fapi2::ReturnCode change_bar_valid_state( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+        const uint8_t i_state)
+{
+    const auto& l_mcs = mss::find_target<fapi2::TARGET_TYPE_MCS>(i_target);
+    fapi2::buffer<uint64_t> l_data;
+
+    FAPI_TRY( mss::getScom(l_mcs, MCS_MCFGP, l_data) );
+    l_data.writeBit<MCS_MCFGP_VALID>(i_state);
+    FAPI_INF("Changing MCS_MCFGP_VALID to %d on %s", i_state, mss::c_str(l_mcs));
+    FAPI_TRY( mss::putScom(l_mcs, MCS_MCFGP, l_data) );
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
 /// @brief Preload the CCS with the EPOW sequence
 /// @param[in] i_target the target associated with this subroutine
 /// @return FAPI2_RC_SUCCESS iff setup was successful
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
index 148273a9a..ad5fb0fa0 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H
@@ -182,15 +182,6 @@ template< fapi2::TargetType T >
 fapi2::ReturnCode self_refresh_exit( const fapi2::Target<T>& i_target );
 
 ///
-/// @brief Disable powerdown mode in rc09
-/// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
-/// @param[in,out] io_inst a vector of CCS instructions we should add to
-/// @return FAPI2_RC_SUCCESS if and only if ok
-///
-fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
-        std::vector< ccs::instruction_t >& io_inst);
-
-///
 /// @brief Load the rcd control words
 /// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
 /// @param[in,out] io_inst a vector of CCS instructions we should add to
@@ -237,6 +228,14 @@ template< fapi2::TargetType T >
 fapi2::ReturnCode post_restore_transition( const fapi2::Target<T>& i_target );
 
 ///
+/// @brief Helper to change the BAR valid state. Consumed by hostboot
+/// @param[in] i_target the target associated with this subroutine
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+fapi2::ReturnCode change_bar_valid_state( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
+        const uint8_t i_state);
+
+///
 /// @brief Preload the CCS with the EPOW sequence
 /// @param[in] i_target the target associated with this subroutine
 /// @return FAPI2_RC_SUCCESS iff setup was successful
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
index 31a69f30d..8928a6f98 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C
@@ -72,7 +72,17 @@ fapi2::ReturnCode mrs_load<TARGET_TYPE_MCA>( const fapi2::Target<TARGET_TYPE_MCA
     for ( const auto& d : find_targets<TARGET_TYPE_DIMM>(i_target) )
     {
         FAPI_DBG("mrs load for %s", mss::c_str(d));
-        FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) );
+
+        // TK - break out the nvdimm stuff into function
+        if (i_nvdimm_workaround)
+        {
+            FAPI_DBG("nvdimm workaround detected. loading mrs for restore sequence");
+            FAPI_TRY( ddr4::mrs_load_nvdimm(d, l_program.iv_instructions) );
+        }
+        else
+        {
+            FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) );
+        }
     }
 
     // We have to configure the CCS engine to let it know which port these instructions are