Adds insert function space helpers for LRDIMM

Change-Id: I2ad35dc3d1a72606b7a0ce72ad407fe7ce10e0c2
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/66086
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: ANDRE A. MARIN <aamarin@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/66097
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>

4 files changed, 352 insertions, 116 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/bcw_load_ddr4.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/bcw_load_ddr4.C
index cf7cd850e..0e0e10a6a 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/bcw_load_ddr4.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/bcw_load_ddr4.C
@@ -44,6 +44,7 @@
 #include <lib/dimm/ddr4/control_word_ddr4.H>
 #include <lib/dimm/ddr4/data_buffer_ddr4.H>
 #include <lib/mss_attribute_accessors.H>
+#include <lib/workarounds/ccs_workarounds.H>
 
 using fapi2::TARGET_TYPE_MCBIST;
 using fapi2::TARGET_TYPE_MCA;
@@ -76,25 +77,6 @@ fapi2::ReturnCode bcw_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
     FAPI_TRY( tdllk(i_target, l_tDLLK), "Failed to get tDLLK for %s in bcw_load_ddr4", mss::c_str(i_target) );
 
     {
-        static const std::vector< cw_data > l_bcw_4bit_data =
-        {
-            // function space #, bcw #, attribute accessor, timing delay
-            { FUNC_SPACE_0,  DQ_RTT_NOM_CW,         eff_dimm_ddr4_bc00,  mss::tmrc() },
-            { FUNC_SPACE_0,  DQ_RTT_WR_CW,          eff_dimm_ddr4_bc01,  mss::tmrc() },
-            { FUNC_SPACE_0,  DQ_RTT_PARK_CW,        eff_dimm_ddr4_bc02,  mss::tmrc() },
-            { FUNC_SPACE_0,  DQ_DRIVER_CW,          eff_dimm_ddr4_bc03,  mss::tmrc() },
-            { FUNC_SPACE_0,  MDQ_RTT_CW,            eff_dimm_ddr4_bc04,  mss::tmrc() },
-            { FUNC_SPACE_0,  MDQ_DRIVER_CW,         eff_dimm_ddr4_bc05,  mss::tmrc() },
-            { FUNC_SPACE_0,  RANK_PRESENCE_CW,      eff_dimm_ddr4_bc07,  mss::tmrc() },
-            { FUNC_SPACE_0,  RANK_SELECTION_CW,     eff_dimm_ddr4_bc08,  mss::tmrc() },
-            { FUNC_SPACE_0,  POWER_SAVING_CW,       eff_dimm_ddr4_bc09,  mss::tmrc() },
-            { FUNC_SPACE_0,  OPERATING_SPEED,       eff_dimm_ddr4_bc0a,  l_tDLLK     },
-            { FUNC_SPACE_0,  VOLT_AND_SLEW_RATE_CW, eff_dimm_ddr4_bc0b,  mss::tmrc() },
-            { FUNC_SPACE_0,  BUFF_TRAIN_MODE_CW,    eff_dimm_ddr4_bc0c,  mss::tmrc() },
-            { FUNC_SPACE_0,  LDQ_OPERATION_CW,      eff_dimm_ddr4_bc0d,  mss::tmrc() },
-            { FUNC_SPACE_0,  PARITY_CW,             eff_dimm_ddr4_bc0e,  mss::tmrc() },
-            { FUNC_SPACE_0,  ERROR_STATUS_CW,       eff_dimm_ddr4_bc0f,  mss::tmrc() },
-        };
 
         // This initialization may be vendor specific.  We might need a different
         // sequence for Montage vs. IDT for example.
@@ -104,33 +86,56 @@ fapi2::ReturnCode bcw_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
         // IDT BCW init
         //
 
-        // We set the 4-bit buffer control words first (they live in function space 0
-        // hw is supposed to default to function space 0 but Just.In.Case.
-        FAPI_TRY( ddr4::function_space_select<0>(i_target, io_inst), "Failed function space select 0", mss::c_str(i_target));
-        FAPI_TRY( control_word_engine<BCW_4BIT>(i_target, l_bcw_4bit_data, l_sim, io_inst) , "Failed control_word_engine",
-                  mss::c_str(i_target));
-
-        // We set our 8-bit buffer control words but have to switch function space
-        // number for different control words.  So it doesn't fit cleanly into a
-        // vector like the 4-bit buffer control words that are all in function space 0
-        // (feels like we should be initializing more control word....)
+        static const std::vector< cw_info > l_bcw_info =
         {
-            cw_data l_data(FUNC_SPACE_6, BUFF_TRAIN_CONFIG_CW, eff_dimm_ddr4_f6bc4x, mss::tmrc());
-            FAPI_TRY( ddr4::function_space_select<6>(i_target, io_inst), "Failed function space select 6", mss::c_str(i_target) );
-            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst), "Failed control_word_engine",
-                      mss::c_str(i_target) );
-        }
+            // function space #, bcw #, attribute accessor, timing delay
+            // We set the 4-bit buffer control words first (they live in function space 0
+            // hw is supposed to default to function space 0 but Just.In.Case.
+            { FUNC_SPACE_0,  FUNC_SPACE_SELECT_CW,  FUNC_SPACE_0,        mss::tmrd() , CW8_DATA_LEN, cw_info::BCW},
+
+            // 4-bit BCW's from here
+            { FUNC_SPACE_0,  DQ_RTT_NOM_CW,         eff_dimm_ddr4_bc00,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  DQ_RTT_WR_CW,          eff_dimm_ddr4_bc01,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  DQ_RTT_PARK_CW,        eff_dimm_ddr4_bc02,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  DQ_DRIVER_CW,          eff_dimm_ddr4_bc03,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  MDQ_RTT_CW,            eff_dimm_ddr4_bc04,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  MDQ_DRIVER_CW,         eff_dimm_ddr4_bc05,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  RANK_PRESENCE_CW,      eff_dimm_ddr4_bc07,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  RANK_SELECTION_CW,     eff_dimm_ddr4_bc08,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  POWER_SAVING_CW,       eff_dimm_ddr4_bc09,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  OPERATING_SPEED,       eff_dimm_ddr4_bc0a,  l_tDLLK     , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  VOLT_AND_SLEW_RATE_CW, eff_dimm_ddr4_bc0b,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  BUFF_TRAIN_MODE_CW,    eff_dimm_ddr4_bc0c,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  LDQ_OPERATION_CW,      eff_dimm_ddr4_bc0d,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  PARITY_CW,             eff_dimm_ddr4_bc0e,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_0,  ERROR_STATUS_CW,       eff_dimm_ddr4_bc0f,  mss::tmrc() , CW4_DATA_LEN, cw_info::BCW},
+
+            // 8-bit BCW's now
+            // Function space 6
+            { FUNC_SPACE_6,  FUNC_SPACE_SELECT_CW,  FUNC_SPACE_6,         mss::tmrd(), CW8_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_6, BUFF_TRAIN_CONFIG_CW,   eff_dimm_ddr4_f6bc4x, mss::tmrc(), CW8_DATA_LEN, cw_info::BCW},
+
+            // Function space 5
+            { FUNC_SPACE_5, FUNC_SPACE_SELECT_CW,   FUNC_SPACE_5,         mss::tmrd(), CW8_DATA_LEN, cw_info::BCW},
+            { FUNC_SPACE_5, BUFF_TRAIN_CONFIG_CW,   eff_dimm_ddr4_f5bc6x, mss::tmrc(), CW8_DATA_LEN, cw_info::BCW},
+
+
+            // So, we always want to know what function space we're in
+            // The way to do that is to always return to one function space
+            // The LR spec recommends that we return to the default function space - function space 0
+            // If the vector is not at our default function space of function space 0, return to function space 0
+            { FUNC_SPACE_0,  FUNC_SPACE_SELECT_CW,  FUNC_SPACE_0,         mss::tmrd(), CW8_DATA_LEN, cw_info::BCW},
+        };
 
-        {
-            cw_data l_data(FUNC_SPACE_5, DRAM_VREF_CW, eff_dimm_ddr4_f5bc6x, mss::tmrc());
-            FAPI_TRY( ddr4::function_space_select<5>(i_target, io_inst), "Failed function space select 5", mss::c_str(i_target) );
-            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst), "Failed control_word_engine",
-                      mss::c_str(i_target) );
-        }
-
-        // Its recommended to always return to the function space
-        // "pointer" back to 0 so we always know where we are starting from
-        FAPI_TRY( ddr4::function_space_select<0>(i_target, io_inst), "Error in bcw_load_ddr4 for function space select 0" );
+        // DES first - make sure those CKE go high and stay there
+        io_inst.push_back(mss::ccs::des_command<TARGET_TYPE_MCBIST>());
+
+        // Issues the CW's
+        FAPI_TRY( control_word_engine(i_target, l_bcw_info, l_sim, io_inst),
+                  "%s Failed control_word_engine", mss::c_str(i_target) );
+
+        // Now, hold the CKE's high, so we don't power down the RCD and re power it back up
+        mss::ccs::workarounds::hold_cke_high(io_inst);
     }
 
 fapi_try_exit:
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/control_word_ddr4.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/control_word_ddr4.H
index e35d143ad..bbef1f8c9 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/control_word_ddr4.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/control_word_ddr4.H
@@ -46,6 +46,9 @@
 
 namespace mss
 {
+///
+/// @brief Control word types
+///
 enum control_word
 {
     // buffer control words
@@ -56,6 +59,10 @@ enum control_word
     RCW_4BIT,
     RCW_8BIT,
 };
+
+// Constexprs to make our lives easier
+static constexpr uint64_t CW4_DATA_LEN = 4;
+static constexpr uint64_t CW8_DATA_LEN = 8;
 }
 
 ///
@@ -80,7 +87,7 @@ class cwTraits< mss::BCW_8BIT >
         // Address bits denoted by A
 
         // Data settings are set by A[7:0]
-        static constexpr uint64_t DATA_LEN = 8;
+        static constexpr uint64_t DATA_LEN = mss::CW8_DATA_LEN;
 
         // Control words are set by A[11:4]
         static constexpr uint64_t WORD_LEN = 4;
@@ -107,7 +114,7 @@ class cwTraits< mss::BCW_4BIT >
         // Address bits denoted by A
 
         // Data settings are set by A[3:0]
-        static constexpr uint64_t DATA_LEN = 4;
+        static constexpr uint64_t DATA_LEN = mss::CW4_DATA_LEN;
 
         // Control words are set by A[11:4]
         // Word length is technically 8
@@ -136,7 +143,7 @@ class cwTraits< mss::RCW_8BIT >
         // Address bits denoted by A
 
         // Data settings are set by A[7:0]
-        static constexpr uint64_t DATA_LEN = 8;
+        static constexpr uint64_t DATA_LEN = mss::CW8_DATA_LEN;
 
         // Control words are set by A[12:8]
         // Word length is technically 5
@@ -165,7 +172,7 @@ class cwTraits< mss::RCW_4BIT >
         // Address bits denoted by A
 
         // Data settings are set by A[3:0]
-        static constexpr uint64_t DATA_LEN = 4;
+        static constexpr uint64_t DATA_LEN = mss::CW4_DATA_LEN;
 
         // Control words are set by A[12:4]
         // Word length is technically 9
@@ -215,7 +222,12 @@ struct cw_data
     }
 
     ///
-    /// @brief default ctor for attribute driven data
+    /// @brief default dtor
+    ///
+    ~cw_data() = default;
+
+    ///
+    /// @brief default ctor
     ///
     cw_data() = default;
 
@@ -267,16 +279,17 @@ struct cw_data
 /// @tparam OT the TargetType of the CCS instruction
 /// @param[in] i_target a DIMM target
 /// @param[in] i_data control word data to send
-/// @param[in,out] io_inst a vector of CCS instructions we should add to
-/// @param[in] i_turn_on_cke flag that states whether we want CKE on for this RCW (defaulted to true)
+/// @param[in] i_sim true if in simulation mode
+/// @param[in] i_turn_on_cke flag that states whether we want CKE on for this RCW
+/// @param[out] o_instruction CCS instruction we created
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
 template< control_word T, typename TT = cwTraits<T>, fapi2::TargetType OT >
 fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
                                       const cw_data& i_data,
                                       const bool i_sim,
-                                      std::vector< ccs::instruction_t<OT> >& io_inst,
-                                      const bool i_turn_on_cke = true)
+                                      const bool i_turn_on_cke,
+                                      ccs::instruction_t<OT>& o_instruction)
 {
     // You're probably asking "Why always turn off CKE's? What is this madness?"
     // Well, due to a vendor sensitivity, we need to have the CKE's off until we run RC09 at the very end
@@ -285,7 +298,7 @@ fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIM
     // Therefore, we want to setup all RCW commands to have CKE's off across both DIMM's
     // We then manually turn on the CKE's associated with a specific DIMM
     constexpr bool CKE_OFF = false;
-    ccs::instruction_t<OT> l_inst =  ccs::rcd_command<OT>(i_target, i_sim, CKE_OFF);
+    ccs::instruction_t<OT> l_inst = ccs::rcd_command<OT>(i_target, i_sim, CKE_OFF);
 
     // Turn on the CKE's for the ranks we're not touching, if it's needed
     // Note: we only have the whole CKE field, not the per DIMM one by default
@@ -332,10 +345,39 @@ fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIM
                  i_data.iv_delay,
                  uint64_t(l_inst.arr0), uint64_t(l_inst.arr1),
                  mss::c_str(i_target));
-
-        io_inst.push_back(l_inst);
     }
 
+    // All setup now, output it
+    o_instruction = l_inst;
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+
+///
+/// @brief Control word engine that sets the CCS instruction
+/// @tparam T the buffer control word type (4 bit or 8 bit)
+/// @tparam TT traits type defaults to cwTraits<T>
+/// @tparam OT the TargetType of the CCS instruction
+/// @param[in] i_target a DIMM target
+/// @param[in] i_data control word data to send
+/// @param[in] i_sim true if in simulation mode
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @param[in] i_turn_on_cke flag that states whether we want CKE on for this RCW (defaulted to true)
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+template< control_word T, typename TT = cwTraits<T>, fapi2::TargetType OT >
+fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                      const cw_data& i_data,
+                                      const bool i_sim,
+                                      std::vector< ccs::instruction_t<OT> >& io_inst,
+                                      const bool i_turn_on_cke = true)
+{
+    ccs::instruction_t<OT> l_inst;
+    FAPI_TRY(control_word_engine<T>(i_target, i_data, i_sim, i_turn_on_cke, l_inst));
+    io_inst.push_back(l_inst);
+
 fapi_try_exit:
     return fapi2::current_err;
 }
@@ -347,6 +389,7 @@ fapi_try_exit:
 /// @tparam OT the TargetType of the CCS instruction
 /// @param[in] i_target a DIMM target
 /// @param[in] i_data_list a vector of control word data to send
+/// @param[in] i_sim true if in simulation mode
 /// @param[in,out] io_inst a vector of CCS instructions we should add to
 /// @param[in] i_turn_on_cke flag that states whether we want CKE on for all RCWs in the vector (defaulted to true)
 /// @return FAPI2_RC_SUCCESS if and only if ok
@@ -358,11 +401,11 @@ fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIM
                                       std::vector< ccs::instruction_t<OT> >& io_inst,
                                       const bool i_turn_on_cke = true)
 {
-    if( i_data_list.empty() )
-    {
-        FAPI_ERR("%s. cw_data vector is empty!", mss::c_str(i_target) );
-        fapi2::Assert(false);
-    }
+    FAPI_ASSERT( !i_data_list.empty(),
+                 fapi2::MSS_EMPTY_VECTOR().
+                 set_FUNCTION(CW_DATA_ENGINE).
+                 set_TARGET(i_target),
+                 "%s. cw_data vector is empty!", mss::c_str(i_target) );
 
     for (const auto& data : i_data_list)
     {
@@ -373,6 +416,170 @@ fapi_try_exit:
     return fapi2::current_err;
 }
 
+/////////////////////////////////////////////////
+// Now for the API related to CW information
+/////////////////////////////////////////////////
+
+///
+/// @class cw_info
+/// @brief class that represents (register/buffer) control word data as well as length and RCW vs BCW
+/// @note used to store all CW data to move the knowledge of 4-bit vs 8-bit to runtime
+///
+struct cw_info
+{
+// Values of A12 which determines BCW vs RCW
+// A12 is 1 for BCW and 0 for RCW
+    static constexpr bool BCW = true;
+    static constexpr bool RCW = false;
+
+    // CW data
+    cw_data iv_cw_data;
+
+    // Data length 4 vs 8 bits
+    uint64_t iv_data_len;
+
+    // Boolean that controls BCW vs RCW - true if BCW
+    bool iv_is_bcw;
+
+    ///
+    /// @brief default dtor
+    ///
+    ~cw_info() = default;
+
+    ///
+    /// @brief default ctor
+    ///
+    cw_info() = default;
+
+    ///
+    /// @brief ctor for attribute driven data
+    /// @param[in] i_func_space the function space number
+    /// @param[in] i_number register control word number
+    /// @param[in] i_func attribute accessor function pointer
+    /// @param[in] i_delay delay in cycles after current RCW
+    /// @param[in] i_data_len data length
+    /// @param[in] i_bcw buffer control word
+    ///
+    cw_info( const uint64_t i_func_space,
+             const uint64_t i_number,
+             fapi2::ReturnCode (*i_func)(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>&, uint8_t&),
+             const uint64_t i_delay,
+             const uint64_t i_data_len,
+             const bool i_bcw ):
+        iv_cw_data(i_func_space, i_number, i_func, i_delay),
+        iv_data_len(i_data_len),
+        iv_is_bcw(i_bcw)
+    {}
+
+    ///
+    /// @brief ctor for custom data
+    /// @param[in] i_func_space the function space number
+    /// @param[in] i_number register control word number
+    /// @param[in] i_data RCW data to write
+    /// @param[in] i_delay delay in cycles after current RCW
+    /// @param[in] i_data_len data length
+    /// @param[in] i_bcw buffer control word
+    ///
+    cw_info( const uint64_t i_func_space,
+             const uint64_t i_number,
+             const uint64_t i_data,
+             const uint64_t i_delay,
+             const uint64_t i_data_len,
+             const bool i_bcw ):
+        iv_cw_data(i_func_space, i_number, i_data, i_delay),
+        iv_data_len(i_data_len),
+        iv_is_bcw(i_bcw)
+    {}
+
+};
+
+///
+/// @brief Control word engine that sets the CCS instruction
+/// @tparam OT the TargetType of the CCS instruction
+/// @param[in] i_target a DIMM target
+/// @param[in] i_info control word data and information about how to send it
+/// @param[in] i_sim true if in simulation mode
+/// @param[in] i_turn_on_cke flag that states whether we want CKE on for this RCW
+/// @param[out] o_instruction CCS instruction we created
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+template< fapi2::TargetType OT >
+fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                      const cw_info& i_info,
+                                      const bool i_sim,
+                                      const bool i_turn_on_cke,
+                                      ccs::instruction_t<OT>& o_instruction)
+{
+    // BCW 4-bit
+    if(i_info.iv_is_bcw && i_info.iv_data_len == CW4_DATA_LEN)
+    {
+        FAPI_TRY(control_word_engine<BCW_4BIT>(i_target, i_info.iv_cw_data, i_sim, i_turn_on_cke, o_instruction));
+    }
+    // BCW 8-bit
+    else if(i_info.iv_is_bcw && i_info.iv_data_len == CW8_DATA_LEN)
+    {
+        FAPI_TRY(control_word_engine<BCW_8BIT>(i_target, i_info.iv_cw_data, i_sim, i_turn_on_cke, o_instruction));
+    }
+    // RCW 4-bit
+    else if(!i_info.iv_is_bcw && i_info.iv_data_len == CW4_DATA_LEN)
+    {
+        FAPI_TRY(control_word_engine<RCW_4BIT>(i_target, i_info.iv_cw_data, i_sim, i_turn_on_cke, o_instruction));
+    }
+    // RCW 8-bit
+    else if(!i_info.iv_is_bcw && i_info.iv_data_len == CW8_DATA_LEN)
+    {
+        FAPI_TRY(control_word_engine<RCW_8BIT>(i_target, i_info.iv_cw_data, i_sim, i_turn_on_cke, o_instruction));
+    }
+    // Error case
+    else
+    {
+        FAPI_ASSERT(false,
+                    fapi2::MSS_INVALID_CW_ENCODING()
+                    .set_CW_DATA_LEN(i_info.iv_data_len)
+                    .set_BCW(i_info.iv_is_bcw)
+                    .set_TARGET(i_target),
+                    "%s data_len: %u and %s are an invalid combination",
+                    mss::c_str(i_target), i_info.iv_data_len, i_info.iv_is_bcw ? "BCW" : "RCW");
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief Control word engine that sets the CCS instruction
+/// @tparam OT the TargetType of the CCS instruction
+/// @param[in] i_target a DIMM target
+/// @param[in] i_info_list a vector of control word data to send
+/// @param[in] i_sim true if in simulation mode
+/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @param[in] i_turn_on_cke flag that states whether we want CKE on for all RCWs in the vector (defaulted to true)
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+template< fapi2::TargetType OT >
+fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                      const std::vector<cw_info>& i_info_list,
+                                      const bool i_sim,
+                                      std::vector< ccs::instruction_t<OT> >& io_inst,
+                                      const bool i_turn_on_cke = true)
+{
+    FAPI_ASSERT( !i_info_list.empty(),
+                 fapi2::MSS_EMPTY_VECTOR().
+                 set_FUNCTION(CW_INFO_ENGINE).
+                 set_TARGET(i_target),
+                 "%s. cw_info vector is empty!", mss::c_str(i_target) );
+
+    for (const auto& l_info : i_info_list)
+    {
+        ccs::instruction_t<OT> l_inst;
+        FAPI_TRY( control_word_engine(i_target, l_info, i_sim, i_turn_on_cke, l_inst) );
+        io_inst.push_back(l_inst);
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
 }// mss
 
 #endif
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/data_buffer_ddr4.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/data_buffer_ddr4.H
index 638122ea6..db26abc36 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/data_buffer_ddr4.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/data_buffer_ddr4.H
@@ -73,6 +73,9 @@ enum db02_def : size_t
     FUNC_SPACE_6 = 6,
     FUNC_SPACE_7 = 7,
 
+    // From DB02 spec - F[3:0]BC7x control word
+    MAX_FUNC_SPACE = 7,
+
     // 4 bit BCWs
     DQ_RTT_NOM_CW = 0x0,
     DQ_RTT_WR_CW = 0x1,
@@ -125,89 +128,106 @@ enum command : size_t
 
 ///
 /// @brief Sets the function space for the BCW
-/// @tparam T the functon space number we want
 /// @param[in] i_target a DIMM target
+/// @param[in] i_func_space the functon space number we want
 /// @param[in,out] io_inst a vector of CCS instructions we should add to
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
-template< uint64_t T >
 inline fapi2::ReturnCode function_space_select(const fapi2::Target< fapi2::TARGET_TYPE_DIMM >& i_target,
+        const uint64_t i_func_space,
         std::vector< ccs::instruction_t<fapi2::TARGET_TYPE_MCBIST> >& io_inst)
 {
-    // From DB02 spec - F[3:0]BC7x control word
-    constexpr size_t MAX_FUNC_SPACE = 7;
-
-    uint8_t l_sim = 0;
-    mss::is_simulation(l_sim);
-
-    // Function spaces are typically user selected.
-    // Sometimes function spaces are selected based on rank number
-    // or MDQ lane number which currently are also user inputs.
-
-    // We assert out for user programming erros
-    static_assert( T <= MAX_FUNC_SPACE, "Invalid function space selection received. Must be <= 7");
+    FAPI_ASSERT(i_func_space <= MAX_FUNC_SPACE,
+                fapi2::MSS_LRDIMM_FUNC_SPACE_OUT_OF_RANGE()
+                .set_TARGET(i_target)
+                .set_MAX_FUNC_SPACE(MAX_FUNC_SPACE)
+                .set_FUNC_SPACE(i_func_space),
+                "%s function space (%u) greater than maximum range (%u)",
+                mss::c_str(i_target), i_func_space, MAX_FUNC_SPACE);
 
     // function space bits for the function space selector are
     // don't cares (XXX).  We choose 0 for simplicity.
-    cw_data l_data( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, T, mss::tmrd() );
+    {
+        uint8_t l_sim = 0;
+        mss::is_simulation(l_sim);
 
-    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst),
-              "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
-              mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number) );
+        cw_data l_data( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, i_func_space, mss::tmrd() );
 
-    // I don't know what already existed in this ccs instruction vector beforehand so
-    // I use the back() method to access the last added ccs instruction of interest
-    FAPI_INF("%s. F%dBC%02lx ccs inst 0x%016llx:0x%016llx, data: %d",
-             mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number),
-             uint64_t(io_inst.back().arr0), uint64_t(io_inst.back().arr1), uint8_t(l_data.iv_data) );
+        FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst),
+                  "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
+                  mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number) );
+
+        // I don't know what already existed in this ccs instruction vector beforehand so
+        // I use the back() method to access the last added ccs instruction of interest
+        FAPI_INF("%s. F%dBC%02lx ccs inst 0x%016llx:0x%016llx, data: %d",
+                 mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number),
+                 uint64_t(io_inst.back().arr0), uint64_t(io_inst.back().arr1), uint8_t(l_data.iv_data) );
+    }
 
 fapi_try_exit:
     return fapi2::current_err;
 }
 
 ///
-/// @brief Sets the function space for the BCW
-/// @param[in] i_target a DIMM target
+/// @brief Inserts a function space selector into a vector of cw_info functions
 /// @param[in] i_func_space the functon space number we want
-/// @param[in,out] io_inst a vector of CCS instructions we should add to
+/// @param[in] i_iterator an iterator to a vector of cw_info structures
+/// @param[in,out] io_vector vector in which to insert the function space selector
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
-inline fapi2::ReturnCode function_space_select(const fapi2::Target< fapi2::TARGET_TYPE_DIMM >& i_target,
-        const uint64_t i_func_space,
-        std::vector< ccs::instruction_t<fapi2::TARGET_TYPE_MCBIST> >& io_inst)
+inline fapi2::ReturnCode insert_function_space_select(const uint64_t i_func_space,
+        const std::vector< cw_info >::iterator& i_iterator,
+        std::vector< cw_info >& io_vector)
 {
-    // From DB02 spec - F[3:0]BC7x control word
-    constexpr size_t MAX_FUNC_SPACE = 7;
-
-    if( i_func_space > MAX_FUNC_SPACE )
+    FAPI_ASSERT(i_func_space <= MAX_FUNC_SPACE,
+                fapi2::MSS_LRDIMM_INSERT_FUNC_SPACE_OUT_OF_RANGE()
+                .set_MAX_FUNC_SPACE(MAX_FUNC_SPACE)
+                .set_FUNC_SPACE(i_func_space),
+                "function space (%u) greater than maximum range (%u)",
+                i_func_space, MAX_FUNC_SPACE);
     {
-        // Function spaces are typically user selected.
-        // Sometimes function spaces are selected based on rank number
-        // or MDQ lane number which currently are also user inputs.
-
-        // We assert out for user programming erros
-        FAPI_ERR("%s. Invalid function space selection: %d, max valid function space is %d",
-                 mss::c_str(i_target), i_func_space, MAX_FUNC_SPACE);
-
-        fapi2::Assert(false);
+        constexpr uint64_t FUNC_SPACE_LEN = 8;
+        cw_info l_info( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, i_func_space, mss::tmrd(), FUNC_SPACE_LEN, cw_info::BCW );
+        io_vector.insert(i_iterator, l_info);
     }
+    return fapi2::FAPI2_RC_SUCCESS;
 
-    uint8_t l_sim = 0;
-    mss::is_simulation(l_sim);
+fapi_try_exit:
+    return fapi2::current_err;
+}
 
-    // function space bits for the function space selector are
-    // don't cares (XXX).  We choose 0 for simplicity.
-    cw_data l_data( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, i_func_space, mss::tmrd() );
+///
+/// @brief Inserts all needed function space selects into a vector of cw_info functions
+/// @param[in,out] io_vector vector in which to insert the function space selector
+/// @return FAPI2_RC_SUCCESS if and only if ok
+///
+inline fapi2::ReturnCode insert_function_space_select(std::vector< cw_info >& io_vector)
+{
+    fapi2::buffer<uint8_t> l_func_space(FUNC_SPACE_0);
 
-    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst),
-              "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
-              mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number) );
+    // Looping on numbers here as using iterators creates memfaults due to the insert
+    for(uint64_t i = 0; i < io_vector.size(); ++i)
+    {
+        // If the current function space is different than our desired one...
+        if(l_func_space != io_vector[i].iv_cw_data.iv_func_space)
+        {
+            // Add in a CW to do the function space change
+            l_func_space = io_vector[i].iv_cw_data.iv_func_space;
+            auto l_it = io_vector.begin() + i;
+            FAPI_TRY(insert_function_space_select(l_func_space, l_it, io_vector));
+        }
+    }
 
-    // I don't know what already existed in this ccs instruction vector beforehand so
-    // I use the back() method to access the last added ccs instruction of interest
-    FAPI_INF("%s. F%dBC%02lx ccs inst 0x%016llx:0x%016llx, data: %d",
-             mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number),
-             uint64_t(io_inst.back().arr0), uint64_t(io_inst.back().arr1), uint8_t(l_data.iv_data) );
+    // So, we always want to know what function space we're in
+    // The way to do that is to always return to one function space
+    // The LR spec recommends that we return to the default function space - function space 0
+    // If the vector is not at our default function space of function space 0, return to function space 0
+    if(l_func_space != FUNC_SPACE_0)
+    {
+        FAPI_TRY(insert_function_space_select(FUNC_SPACE_0, io_vector.end(), io_vector));
+    }
+
+    return fapi2::FAPI2_RC_SUCCESS;
 
 fapi_try_exit:
     return fapi2::current_err;
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H b/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
index 8238a888c..bbd083730 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/shared/mss_const.H
@@ -196,6 +196,10 @@ enum ffdc_function_codes
 
     // eff_dimm.C
     PRIMARY_STACK_TYPE = 116,
+
+    // CW engine information
+    CW_DATA_ENGINE = 117,
+    CW_INFO_ENGINE = 118,
 };
 
 enum states