From 3440e65b32f2c84f79b5cabfe939337d1e41e38f Mon Sep 17 00:00:00 2001 From: Stephen Glancy Date: Wed, 18 Oct 2017 13:46:41 -0500 Subject: Adds PDA support PDA support will be needed for: 1) NVDIMM support (re-programming WR VREF) 2) PDA shmoo Change-Id: Ib4f910721a47d69897daac5fa42ac5bd73464237 Original-Change-Id: Ic49bbbbe907ee9609ee26f6152de8949cdb999a3 Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/48575 Tested-by: FSP CI Jenkins Reviewed-by: ANDRE A. MARIN Tested-by: Jenkins Server Tested-by: HWSV CI Reviewed-by: Louis Stermole Tested-by: Hostboot CI Reviewed-by: Jennifer A. Stofer Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/51127 Tested-by: Jenkins OP Build CI Tested-by: Jenkins OP HW Reviewed-by: Christian R. Geddes --- .../p9/procedures/hwp/memory/lib/dimm/ddr4/pda.C | 436 +++++++++++++++++++++ .../p9/procedures/hwp/memory/lib/dimm/ddr4/pda.H | 330 ++++++++++++++++ .../hwp/memory/lib/workarounds/ccs_workarounds.C | 118 ++++++ .../hwp/memory/lib/workarounds/ccs_workarounds.H | 61 +++ 4 files changed, 945 insertions(+) diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.C index 7e1ecf55e..5117cfe31 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.C @@ -22,3 +22,439 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ + +/// +/// @file pda.C +/// @brief Code to support per-DRAM addressability (PDA) +/// +// *HWP HWP Owner: Stephen Glancy +// *HWP HWP Backup: Louis Stermole +// *HWP Team: Memory +// *HWP Level: 3 +// *HWP Consumed by: FSP:HB Memory Lab + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mss +{ + +namespace ddr4 +{ + +namespace pda +{ + +const std::vector> pdaBitTraits::BIT_MAP = +{ + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N3}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N3}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N3}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N3}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_4, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_4, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1}, +}; + +const std::vector> pdaBitTraits::BIT_MAP = +{ + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_1, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_2, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_3, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2}, + + {MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_4, MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0}, +}; + +/// +/// @brief Helper function for changing the DRAM bit +/// @tparam W the DRAM width +/// @tparam TT the DRAM width traits class +/// @param[in] i_target - the MCA target +/// @param[in] i_dram - the DRAM on which to operate +/// @param[in] i_state - the state to write the bit(s) to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +template< uint8_t W, typename TT = pdaBitTraits > +fapi2::ReturnCode change_dram_bit_helper( const fapi2::Target& i_target, + const uint64_t i_dram, + const mss::states& i_state) +{ + fapi2::buffer l_data; + + // Note: the following avoids "undefined reference to" errors due to the set max dram below + // The use of traits and a const reference messes with it + constexpr auto NUM_DRAM = TT::NUM_DRAMS; + + // Check bounds + FAPI_ASSERT(i_dram < NUM_DRAM, + fapi2::MSS_PDA_DRAM_OUT_OF_RANGE(). + set_MCA_TARGET(i_target). + set_DRAM(i_dram). + set_MAX_DRAM(NUM_DRAM), + "%s was passed DRAM value of %lu which is not below the max value of %lu", + mss::c_str(i_target), i_dram, NUM_DRAM); + + FAPI_TRY(mss::getScom(i_target, TT::BIT_MAP[i_dram].first, l_data)); + FAPI_TRY(l_data.writeBit(i_state, TT::BIT_MAP[i_dram].second, TT::NUM_BITS)); + FAPI_TRY(mss::putScom(i_target, TT::BIT_MAP[i_dram].first, l_data)); + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Writes the data bit enable for the properly inputted DRAM +/// @param[in] i_target - the MCA target +/// @param[in] i_dram - the DRAM on which to operate +/// @param[in] i_state - the state to write the bit(s) to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode change_dram_bit( const fapi2::Target& i_target, + const uint64_t i_dram, + const mss::states& i_state) +{ + uint8_t l_dram_width[MAX_DIMM_PER_PORT] = {0}; + FAPI_TRY(mss::eff_dram_width(i_target, &(l_dram_width[0])), "Failed to get the DRAM's width for %s", + mss::c_str(i_target)); + + { + const auto& l_dimms = mss::find_targets(i_target); + FAPI_ASSERT((l_dram_width[0] == fapi2::ENUM_ATTR_EFF_DRAM_WIDTH_X4) || + (l_dram_width[0] == fapi2::ENUM_ATTR_EFF_DRAM_WIDTH_X8), + fapi2::MSS_INVALID_DRAM_WIDTH(). + set_DIMM_TARGET(l_dimms[0]). + set_DRAM_WIDTH(l_dram_width[0]), + "%s DRAM width was not x4 or x8 - %lu", mss::c_str(i_target), l_dram_width[0]); + } + + // We only need to check DIMM 0 due to the plug rules + // x4 DIMM + if(l_dram_width[0] == fapi2::ENUM_ATTR_EFF_DRAM_WIDTH_X4) + { + FAPI_TRY(change_dram_bit_helper(i_target, i_dram, i_state)); + } + // x8 DIMM. The else is ok here as we checked the widths above + else + { + FAPI_TRY(change_dram_bit_helper(i_target, i_dram, i_state)); + } + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Configures all DRAM level configuration bits to the inputted settings +/// @param[in] i_target a fapi2::Target MCA +/// @param[in] i_state - OFF_N - 1's, ON_N - 0's +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note PDA is LOW enable, so 0's means ON. ON will configure the register to 0's - using OFF/ON_N +/// +fapi2::ReturnCode blast_dram_config( const fapi2::Target& i_target, const mss::states& i_state ) +{ + typedef mss::dp16Traits TT; + + std::vector> l_reg_data; + + // Gets the register data + FAPI_TRY(mss::scom_suckah(i_target, TT::DATA_BIT_ENABLE1, l_reg_data)); + + // Loops and modifies the data + for(auto& l_data : l_reg_data) + { + // Remember 1 = OFF + set_dram_enable(l_data, i_state); + set_dram_enable(l_data, i_state); + set_dram_enable(l_data, i_state); + set_dram_enable(l_data, i_state); + } + + // Blasts the data back out + FAPI_TRY(mss::scom_blastah(i_target, TT::DATA_BIT_ENABLE1, l_reg_data)); + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Configures PDA timings +/// @param[in] i_target a fapi2::Target MCA +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode configure_timings( const fapi2::Target& i_target ) +{ + // Fun fact, we're hitting all of the bits in this reg, no need for RMW + fapi2::buffer l_data; + + // So we want to: + // 1) Turn on the PDA on MRS bit + // 2) Have a 0 delay between the MRS being sent and starting the 0/1 latching + // 3) Hold the delay for as long as possible (safer and easier than figuring out how long to hold the values) + mss::wc::set_pda_mode(l_data, mss::states::ON); + mss::wc::set_pda_dq_on_delay(l_data, START_DELAY_VALUE); + mss::wc::set_pda_dq_off_delay(l_data, HOLD_DELAY_VALUE); + + // Set that reg + FAPI_TRY(mss::wc::write_config3(i_target, l_data)); + +fapi_try_exit: + return fapi2::current_err; +} + + +/// +/// @brief Adds a PDA enable command +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in,out] io_inst the CCS instructions to update +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode add_enable( const fapi2::Target& i_target, + const uint64_t i_rank, + std::vector< ccs::instruction_t >& io_inst ) +{ + mss::ddr4::mrs03_data l_mrs03( i_target, fapi2::current_err ); + FAPI_TRY( fapi2::current_err, "%s Unable to construct MRS03 data from attributes", mss::c_str(i_target)); + + // Overrides the PDA value to be enabled + l_mrs03.iv_pda = fapi2::ENUM_ATTR_EFF_PER_DRAM_ACCESS_ENABLE; + + FAPI_TRY( mss::mrs_engine(i_target, l_mrs03, i_rank, DELAY, io_inst) ); + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Enters into and configures PDA mode +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode enter( const fapi2::Target& i_target, + const uint64_t i_rank ) +{ + ccs::program l_program; + + const auto& l_mca = mss::find_target(i_target); + + FAPI_TRY( add_enable(i_target, i_rank, l_program.iv_instructions) ); + + FAPI_TRY( ccs::execute(mss::find_target(i_target), + l_program, + l_mca), + "unable to execute CCS for MR03 rank %d %s", + i_rank, mss::c_str(i_target) ); + + // Now sets up all of the PDA regs now that we are in PDA mode + FAPI_TRY(configure_timings(l_mca)); + FAPI_TRY(blast_dram_config(l_mca, mss::states::OFF_N)); + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Adds a PDA disable command +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in,out] io_inst the instructions +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode add_disable( const fapi2::Target& i_target, + const uint64_t i_rank, + std::vector< ccs::instruction_t >& io_inst ) +{ + mss::ddr4::mrs03_data l_mrs03( i_target, fapi2::current_err ); + FAPI_TRY( fapi2::current_err, "%s Unable to construct MRS03 data from attributes", mss::c_str(i_target)); + + // Overrides the PDA value to be disabled + l_mrs03.iv_pda = fapi2::ENUM_ATTR_EFF_PER_DRAM_ACCESS_DISABLE; + + FAPI_TRY( mss::mrs_engine(i_target, l_mrs03, i_rank, DELAY, io_inst) ); + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Exits out of and disables PDA mode +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode exit( const fapi2::Target& i_target, + const uint64_t i_rank ) +{ + ccs::program l_program; + + const auto& l_mca = mss::find_target(i_target); + + // We need everyone to exit PDA mode, so all of them are all ON + FAPI_TRY(blast_dram_config(l_mca, mss::states::ON_N)); + + FAPI_TRY( add_disable(i_target, i_rank, l_program.iv_instructions) ); + + FAPI_TRY( mss::ccs::workarounds::exit(i_target, i_rank, l_program) ); + + FAPI_TRY( ccs::execute(mss::find_target(i_target), + l_program, + l_mca), + "unable to execute CCS for MR03 PDA exit rank %d %s", + i_rank, mss::c_str(i_target) ); + + // Disables PDA mode + { + fapi2::buffer l_data; + FAPI_TRY(mss::wc::read_config3(l_mca, l_data)); + mss::wc::set_pda_mode(l_data, mss::states::OFF); + FAPI_TRY(mss::wc::write_config3(l_mca, l_data)); + } + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Performs a PDA WR VREF latch +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in] i_mrs the MRS data to update +/// @param[in] i_drams the DRAM to update +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note A PDA latch of WR VREF settings is the most common PDA operations +/// This function adds a bit of fanciness (compression) to speed up the overall runtime +/// +fapi2::ReturnCode execute_wr_vref_latch( const fapi2::Target& i_target, + const uint64_t i_rank, + const mss::ddr4::mrs06_data& i_mrs, + const std::vector& i_drams ) +{ + const auto& l_mca = mss::find_target(i_target); + + // If the commands passed in are empty, simply exit + FAPI_ASSERT((!i_drams.empty()), + fapi2::MSS_EMPTY_PDA_VECTOR(). + set_PROCEDURE(mss::ffdc_function_codes::PDA_WR_VREF_LATCH_VECTOR), + "%s PDA commands vector is empty, exiting", mss::c_str(i_target)); + + // Enable all individual DRAM + for(const auto l_dram : i_drams) + { + FAPI_TRY(change_dram_bit( l_mca, l_dram, mss::states::ON_N)); + } + + // Issue MRS commands + { + ccs::program l_program; + + FAPI_TRY(mss::ddr4::add_latch_wr_vref_commands( i_target, + i_mrs, + i_rank, + l_program.iv_instructions)); + + FAPI_TRY( ccs::execute(mss::find_target(i_target), + l_program, + l_mca), + "unable to execute CCS for MR06 rank %d %s", + i_rank, mss::c_str(i_target) ); + } + + // Disable all individual DRAM + for(const auto l_dram : i_drams) + { + FAPI_TRY(change_dram_bit( l_mca, l_dram, mss::states::OFF_N)); + } + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Performs a PDA WR VREF latch +/// @param[in] i_commands the PDA commands to issue and DRAM +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note A PDA latch of WR VREF settings is the most common PDA operations +/// This function adds a bit of fanciness (compression) to speed up the overall runtime +/// +fapi2::ReturnCode execute_wr_vref_latch( const commands& i_commands ) +{ + // If the commands passed in are empty, simply exit + FAPI_ASSERT((!i_commands.empty()), + fapi2::MSS_EMPTY_PDA_VECTOR(). + set_PROCEDURE(mss::ffdc_function_codes::PDA_WR_VREF_LATCH_CONTAINER), + "PDA commands map is empty, exiting"); + + // Loop until all commands have been issued + for(const auto& l_commands : i_commands.get()) + { + // PDA targetting information - stores both the DIMM and rank in a pair + const auto& l_target_info = l_commands.first; + const auto& l_dimm = l_target_info.first; + const auto l_rank = l_target_info.second; + + // The PDA commands consist of MRS's and their associated DRAM's + const auto& l_pda_commands = l_commands.second; + + // First, enter into PDA mode + FAPI_TRY(enter(l_dimm, l_rank)); + + // Now loops through all of the MRS and DRAM + for(const auto& l_command : l_pda_commands) + { + const auto& l_mrs = l_command.first; + const auto& l_drams = l_command.second; + FAPI_TRY(execute_wr_vref_latch( l_dimm, + l_rank, + l_mrs, + l_drams )); + } + + // Finally, exit out of PDA + FAPI_TRY(exit(l_dimm, l_rank)); + } + +fapi_try_exit: + return fapi2::current_err; +} + +} // ns pda + +} // ns ddr4 + +} // ns mss diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.H index bb514b283..786ecad54 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.H +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/pda.H @@ -22,3 +22,333 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ + +/// +/// @file pda.H +/// @brief Code to support per-DRAM addressability (PDA) +/// +// *HWP HWP Owner: Stephen Glancy +// *HWP HWP Backup: Louis Stermole +// *HWP Team: Memory +// *HWP Level: 3 +// *HWP Consumed by: FSP:HB Memory Lab + +#ifndef _MSS_PDA_H_ +#define _MSS_PDA_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace mss +{ + +namespace ddr4 +{ + +namespace pda +{ + +/// +/// @brief Holds all PDA specific constants +/// +enum consts : uint64_t +{ + // Note: the delay is taken from the PDA timing register's maximum value 2^6 + MAX_PDA_REG = 64, + // 10 cycles for safety + SAFETY = 10, + // Total delay is your register max + safety + DELAY = MAX_PDA_REG + SAFETY, + + // PDA on/off settings + START_DELAY_VALUE = 0, + HOLD_DELAY_VALUE = 0xff, + + // Each enable is just 1 bit and there are 4 per reg. Adding a length here to make the code look better + NUM_DRAM_PER_REG = 4, + + // DRAM bit constants + DRAM0 = MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N0, + DRAM1 = MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N1, + DRAM2 = MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N2, + DRAM3 = MCA_DDRPHY_DP16_DATA_BIT_ENABLE1_P0_0_01_MRS_CMD_N3, + DRAM_START = DRAM0, +}; + +/// +/// @brief Write bit traits class +/// @tparam W the DRAM's width +/// +template < uint8_t W > +class pdaBitTraits; + +/// +/// @brief Write bit for PDA specialization for x4 devices +/// +template < > +class pdaBitTraits +{ + public: + // One nibble is needed - the register has one nibble per bit, so one bit + static constexpr uint64_t NUM_BITS = 1; + static constexpr uint64_t NUM_DRAMS = MAX_DRAMS_X4; + static const std::vector> BIT_MAP; +}; + +/// +/// @brief Write bit for PDA specialization for x4 devices +/// +template < > +class pdaBitTraits +{ + public: + // Two nibbles are needed - the register has one nibble per bit, so two bits + static constexpr uint64_t NUM_BITS = 2; + static constexpr uint64_t NUM_DRAMS = MAX_DRAMS_X8; + static const std::vector> BIT_MAP; +}; + +/// +/// @brief Enables or disables a given DRAM +/// @tparam D the DRAM to enable or disable +/// @param[in,out] io_data the data buffer to modify +/// @param[in] i_state the state to modify the buffer to +/// +template< uint64_t D > +inline void set_dram_enable( fapi2::buffer& io_data, const mss::states& i_state) +{ + io_data.writeBit(i_state); +} + +/// +/// @brief Configures PDA timings +/// @param[in] i_target a fapi2::Target MCA +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode configure_timings( const fapi2::Target& i_target ); + +/// +/// @brief Configures all DRAM level configuration bits to the inputted settings +/// @param[in] i_target a fapi2::Target MCA +/// @param[in] i_state - OFF - 1's, ON - 0's +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note PDA is LOW enable, so 0's means ON. ON will configure the register to 0's +/// +fapi2::ReturnCode blast_dram_config( const fapi2::Target& i_target, + const mss::states& i_state ); + +/// +/// @brief Adds a PDA enable command +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in,out] io_inst the CCS instructions to update +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode add_enable( const fapi2::Target& i_target, + const uint64_t i_rank, + std::vector< ccs::instruction_t >& io_inst ); + +/// +/// @brief Enters into and configures PDA mode +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode enter( const fapi2::Target& i_target, + const uint64_t i_rank ); + +/// +/// @brief Adds a PDA disable command +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in,out] io_inst the CCS instructions to update +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode add_disable( const fapi2::Target& i_target, + const uint64_t i_rank, + std::vector< ccs::instruction_t >& io_inst ); + +/// +/// @brief Exits out of and disables PDA mode +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode exit( const fapi2::Target& i_target, + const uint64_t i_rank ); + +/// +/// @brief Writes the data bit enable for the properly inputted DRAM +/// @param[in] i_target - the MCA target +/// @param[in] i_dram - the DRAM on which to operate +/// @param[in] i_state - the state to write the bit(s) to +/// @param[in] i_delay the delay to use for the PDA command +/// @note PDA is LOW enable, so 0's means ON. ON will configure the register to 0's +/// +fapi2::ReturnCode change_dram_bit( const fapi2::Target& i_target, + const uint64_t i_dram, + const mss::states& i_state); + +/// +/// @brief PDA commands container that also has compression +/// @tparam D the MRS command type +/// +template< typename D > +class commands +{ + public: + // Typdefs to make the code more readable + typedef std::pair, uint64_t> rank_target; + typedef std::map > mrs_drams; + + /// + /// @brief Base constructor + /// + commands() = default; + + /// + /// @brief Base destructor + /// + ~commands() = default; + + /// + /// @brief Adds in a PDA command if need be + /// @param[in] i_target the MCS target + /// @param[in] i_rank the rank + /// @param[in] i_mrs_container the MRS container to add + /// @param[in] i_dram the DRAM to issue the PDA command to + /// @return FAPI2_RC_SUCCESS if and only if ok + /// + fapi2::ReturnCode add_command( const fapi2::Target& i_target, + const uint64_t i_rank, + const D& i_mrs, + const uint64_t i_dram ) + { + fapi2::Target l_dimm; + FAPI_TRY( mss::rank::get_dimm_target_from_rank(i_target, + i_rank, + l_dimm)); + + FAPI_TRY(add_command(l_dimm, i_rank, i_mrs, i_dram)); + + fapi_try_exit: + return fapi2::current_err; + } + + /// + /// @brief Adds in a PDA command if need be + /// @param[in] i_target the DIMM target + /// @param[in] i_rank the rank + /// @param[in] i_mrs_container the MRS container to add + /// @param[in] i_dram the DRAM to issue the PDA command to + /// @return FAPI2_RC_SUCCESS if and only if ok + /// + fapi2::ReturnCode add_command( const fapi2::Target& i_target, + const uint64_t i_rank, + const D& i_mrs, + const uint64_t i_dram ) + { + fapi2::current_err = fapi2::FAPI2_RC_SUCCESS; + + // Check for a valid rank + FAPI_ASSERT(mss::rank::is_rank_on_dimm(i_target, i_rank), + fapi2::MSS_INVALID_RANK(). + set_MCA_TARGET(mss::find_target(i_target)). + set_RANK(i_rank). + set_FUNCTION(mss::ffdc_function_codes::PDA_ADD_COMMAND), + "%s does not have rank %lu", mss::c_str(i_target), i_rank); + + iv_commands[ {i_target, i_rank}][i_mrs].push_back(i_dram); + + fapi_try_exit: + return fapi2::current_err; + } + + /// + /// @brief Checks whether the map is empty + /// @return bool true if empty + /// + inline bool empty() const + { + return iv_commands.empty(); + } + + /// + /// @brief Clears the commands + /// + inline void clear() + { + iv_commands.clear(); + } + + /// + /// @brief Returns the command information + /// @return iv_commands + /// + inline const typename std::map& get() const + { + return iv_commands; + } + + private: + // The following is a map of targets/DIMM as the key to a map of + // the MRS command as the key to the DRAM's to toggle. An explanation as to the data structure is included below + + // PDA compression is a little complex, but is organized to allow us to minimize the number of commands run + // Each individual map is designed to further minimize the number of commands run + // The compressed commands consist of a map within a map + // The outside map, maps the DIMM/rank to the MRS command and DRAM's that need to be run + // Basically, it's a list of a specific rank target with all the commands that need to be run + // The rank-specific target information allows us to just issue the enter/exit commands for PDA for each rank once + // The MRS commands to the DRAM are then looped over in the inside loop + // The inside map has a key of the MRS and a value of a vector of DRAM's to issue the MRS to + // CCS does not allow the user to toggle the DQ during an MRS command + // The DQ information is stored in separate registers in the PHY + // What this means for issuing the commands is that we have to issue an invocation of CCS for each different MRS command we issue + // We can issue a single MRS command for multiple DRAM's however + // Each invocation of CCS creates a noticable increase in time, as the registers need to be configured, CCS needs to be started, and we need to poll for done + // By only entering into PDA on a DIMM-rank once and by issuing the PDA MRS's to multiple DRAM's at a time, we can save a lot of runtime + // Note: in shmoo, adding the compression reduced runtime from about 13 minutes down to 3 minutes + typename std::map iv_commands; +}; + +/// +/// @brief Performs a PDA WR VREF latch +/// @param[in] i_target a fapi2::Target DIMM +/// @param[in] i_rank the rank to send to +/// @param[in] i_mrs the MRS data to update +/// @param[in] i_drams the DRAM to update +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note A PDA latch of WR VREF settings is the most common PDA operations +/// This function adds a bit of fanciness (compression) to speed up the overall runtime +/// +fapi2::ReturnCode execute_wr_vref_latch( const fapi2::Target& i_target, + const uint64_t i_rank, + const mss::ddr4::mrs06_data& i_mrs, + const std::vector& i_drams ); + +/// +/// @brief Performs a PDA WR VREF latch +/// @param[in] i_commands the PDA commands to issue and DRAM +/// @return FAPI2_RC_SUCCESS if and only if ok +/// @note A PDA latch of WR VREF settings is the most common PDA operations +/// This function adds a bit of fanciness (compression) to speed up the overall runtime +/// +fapi2::ReturnCode execute_wr_vref_latch( const commands& i_commands ); + +} // ns pda + +} // ns ddr4 + +} // ns mss + +#endif diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C index 896f341ff..bc4b791b7 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.C @@ -22,3 +22,121 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ + +/// +/// @file ccs_workarounds.H +/// @brief Contains CCS workarounds +/// +// *HWP HWP Owner: Stephen Glancy +// *HWP HWP Backup: Louis Stermole +// *HWP Team: Memory +// *HWP Level: 3 +// *HWP Consumed by: FSP:HB Memory Lab + +#include +#include +#include +#include + +namespace mss +{ + +namespace ccs +{ + +namespace workarounds +{ + +/// +/// @brief Re-enables PDA mode on a given rank in the shadow registers +/// @param[in] i_target - the MCA target on which to operate +/// @param[in] i_rank - the rank on which to operate +/// @return fapi2::ReturnCode - SUCCESS iff everything executes successfully +/// +fapi2::ReturnCode enable_pda_shadow_reg( const fapi2::Target& i_target, + const uint64_t i_rank ) +{ + static const std::vector RP_TO_REG = + { + MCA_DDRPHY_PC_MR3_PRI_RP0_P0, + MCA_DDRPHY_PC_MR3_PRI_RP1_P0, + MCA_DDRPHY_PC_MR3_PRI_RP2_P0, + MCA_DDRPHY_PC_MR3_PRI_RP3_P0, + }; + + uint64_t l_rp = 0; + FAPI_TRY( mss::rank::get_pair_from_rank( i_target, i_rank, l_rp ), "%s failed to get pair from rank %lu", + mss::c_str(i_target), i_rank ); + + // Reads, modifies, and writes the value back out + { + // The bits in the shadow register are one block, we only want to set the PDA enable bit, which corresponds to bit 59 + constexpr uint64_t PDA_BIT = 59; + fapi2::buffer l_data; + + // Read + FAPI_TRY(mss::getScom(i_target, RP_TO_REG[l_rp], l_data)); + + // Modify + l_data.setBit(); + + // Write + FAPI_TRY(mss::putScom(i_target, RP_TO_REG[l_rp], l_data)); + } + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Issues the PDA exit command +/// @param[in] i_target - the DIMM target on which to operate +/// @param[in] i_rank - the rank on which to operate +/// @param[in,out] io_program - the CCS program +/// @return fapi2::ReturnCode - SUCCESS iff everything executes successfully +/// @note The PHY traps both the a-side and b-side MRS's into the same shadow register +/// After the a-side MRS exits PDA, the b-side MRS will not be taken out of PDA mode +/// To workaround this problem, a-side MRS is issued, then the shadow register is modified to have PDA mode enabled +/// Then the b-side MRS is issued +/// +fapi2::ReturnCode exit( const fapi2::Target& i_target, + const uint64_t i_rank, + ccs::program& io_program ) +{ + const auto& l_mca = mss::find_target(i_target); + // Issues A-side MRS + { + auto l_a_side = io_program; + l_a_side.iv_instructions.clear(); + l_a_side.iv_instructions.push_back(io_program.iv_instructions[0]); + + FAPI_TRY( ccs::execute(mss::find_target(i_target), + l_a_side, + l_mca), + "unable to execute CCS for MR03 a-side PDA exit rank %d %s", + i_rank, mss::c_str(i_target) ); + } + + // Re-enable PDA mode in the PHY + { + FAPI_TRY(enable_pda_shadow_reg(l_mca, i_rank)); + } + + // Sets up the B-side MRS - the outside code will issue it + // This allows the workaround to be encapuslated and the exit code to function properly for cases where the workaround should not be executed + { + auto l_b_side = io_program; + l_b_side.iv_instructions.clear(); + l_b_side.iv_instructions.push_back(io_program.iv_instructions[1]); + io_program = l_b_side; + } + +fapi_try_exit: + return fapi2::current_err; +} + +} // ns workarounds + +} // ns ccs + +} // ns mss diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H index e85c8ea44..c39187b37 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H +++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/ccs_workarounds.H @@ -22,3 +22,64 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ + +/// +/// @file ccs_workarounds.H +/// @brief Contains CCS workarounds +/// +// *HWP HWP Owner: Stephen Glancy +// *HWP HWP Backup: Louis Stermole +// *HWP Team: Memory +// *HWP Level: 3 +// *HWP Consumed by: FSP:HB Memory Lab + +#ifndef _CCS_WORKAROUNDS_H +#define _CCS_WORKAROUNDS_H + +#include +#include + +#include +#include +#include + +namespace mss +{ + +namespace ccs +{ + +namespace workarounds +{ + +/// +/// @brief Issues the PDA exit command +/// @param[in] i_target - the DIMM target on which to operate +/// @param[in] i_rank - the rank on which to operate +/// @param[in,out] io_program - the CCS program +/// @return fapi2::ReturnCode - SUCCESS iff everything executes successfully +/// @note The PHY traps both the a-side and b-side MRS's into the same shadow register +/// After the a-side MRS exits PDA, the b-side MRS will not be taken out of PDA mode +/// To workaround this problem, a-side MRS is issued, then the shadow register is modified to have PDA mode enabled +/// Then the b-side MRS is issued +/// +fapi2::ReturnCode exit( const fapi2::Target& i_target, + const uint64_t i_rank, + ccs::program& io_program ); + +/// +/// @brief Re-enables PDA mode on a given rank in the shadow registers +/// @param[in] i_target - the MCA target on which to operate +/// @param[in] i_rank - the rank on which to operate +/// @return fapi2::ReturnCode - SUCCESS iff everything executes successfully +/// +fapi2::ReturnCode enable_pda_shadow_reg( const fapi2::Target& i_target, + const uint64_t i_rank ); + +} // ns workarounds + +} // ns ccs + +} // ns mss + +#endif -- cgit v1.2.1