From 158c1ca0eaa97d496646830610a0b0038602c527 Mon Sep 17 00:00:00 2001 From: Peng Fei GOU Date: Wed, 26 Apr 2017 02:12:04 -0500 Subject: p9_throttle_sync -- initial version for Cumulus 1) Mostly copied from Nimbus settings. 2) Add findNumDimms() for dimm numbers. 3) Add MCMODE0 bit 27/28 programming for Nimbus DD2 and Cumulus. 4) No HW397255 workaround for Nimbus DD2 and Cumulus 5) Generalize throttleSync(), progMaster() and progMCMODE0(). Change-Id: I74d96f4113cb2a33eaba6e1966b17aac693c9c4e Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39695 Reviewed-by: Joseph J. McGill Reviewed-by: Thi N. Tran Tested-by: Jenkins Server Tested-by: PPE CI Tested-by: Hostboot CI Reviewed-by: Jennifer A. Stofer Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39712 Reviewed-by: Hostboot Team Tested-by: Jenkins OP Build CI Tested-by: FSP CI Jenkins Reviewed-by: Daniel M. Crowell --- .../include/p9_mc_scom_addresses_fld_fixes.H | 4 + .../p9/procedures/hwp/nest/p9_throttle_sync.C | 269 ++++++++++++++++----- 2 files changed, 206 insertions(+), 67 deletions(-) (limited to 'src') diff --git a/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H b/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H index 69deff6da..ef6af5669 100644 --- a/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H +++ b/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H @@ -68,6 +68,10 @@ REG64_FLD( MCS_MCMODE0_GROUP_INTERLEAVE_GRANULARITY , 52 , SH_UN 0 ); REG64_FLD( MCS_MCMODE0_GROUP_INTERLEAVE_GRANULARITY_LEN , 4 , SH_UNT_MCS , SH_ACS_SCOM_RW , 0 ); +REG64_FLD( MCS_MCMODE0_DISABLE_MC_SYNC , 27 , SH_UNT_MCS , SH_ACS_SCOM_RW , + 0 ); +REG64_FLD( MCS_MCMODE0_DISABLE_MC_PAIR_SYNC , 28 , SH_UNT_MCS , SH_ACS_SCOM_RW , + 0 ); diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C b/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C index 28882ad36..e2715b72f 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C +++ b/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C @@ -44,34 +44,178 @@ #include #include #include +#include ///---------------------------------------------------------------------------- /// Constant definitions ///---------------------------------------------------------------------------- const uint8_t SUPER_SYNC_BIT = 14; const uint8_t MAX_MC_SIDES_PER_PROC = 2; // MC01, MC23 +const uint8_t MAX_MC_PER_PROC = 4; // MC0, MC1, MC2, MC3 +const uint8_t MAX_MC_PER_SIDE = 2; // MC0, MC1 or MC2, MC3 -// Structure that holds the potential master MCS for a MC side (MC01/MC23) +// Structure that holds the potential master MCS or MI for a MC side (MC01/MC23) +template struct mcSideInfo_t { - bool masterMcsFound = false; - // Master MCS for this MC side - fapi2::Target masterMcs; + bool masterMcFound = false; + + // Master MC for this MC side + fapi2::Target masterMc; }; +/// +/// @brief Find out how many DIMMS are connected to a MCS or MI target +/// +/// @tparam T template parameter, passed in targets. +/// @param[in] i_miTarget The MI target to be programmed as master +/// +/// @return uint8_t number of DIMMs +/// +template< fapi2::TargetType T> +uint8_t findNumDimms(const fapi2::Target& i_mcTarget); + +template<> +uint8_t findNumDimms(const fapi2::Target& i_miTarget) +{ + FAPI_DBG("Entering findNumDimms"); + auto l_dmiChiplets = i_miTarget.getChildren(); + + uint8_t l_num_dimms = 0; + + for (auto l_dmi : l_dmiChiplets) + { + auto l_memBufs = l_dmi.getChildren(); + + if (l_memBufs.size() > 0) + { + l_num_dimms++; + } + } + + FAPI_DBG("Exiting findNumDimms"); + return l_num_dimms; +} + +template<> +uint8_t findNumDimms(const fapi2::Target& i_mcsTarget) +{ + + FAPI_DBG("Entering findNumDimms"); + std::vector< fapi2::Target > l_dimms = + mss::find_targets(i_mcsTarget); + + FAPI_DBG("Exiting findNumDimms"); + return l_dimms.size(); +} + +/// +/// @brief Program MCMODE0 based on the functional targets +/// +/// @tparam T template parameter, passed in targets. +/// @param[in] i_mcTarget The MC target to be programmed +/// @param[in] i_mcTargets Other MC targets. +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +template< fapi2::TargetType T> +fapi2::ReturnCode progMCMODE0( + fapi2::Target& i_mcTarget, + const std::vector< fapi2::Target >& i_mcTargets) +{ + FAPI_DBG("Entering progMCMODE0"); + // -------------------------------------------------------------- + // Setup MCMODE0 for disabling MC SYNC to other-side and same-side + // partner unit. + // BIT27: set if other-side MC is non-functional, 0<->2, 1<->3 + // BIT28: set if same-side MC is non-functional, 0<->1, 2<->3 + // -------------------------------------------------------------- + fapi2::buffer l_scomData(0); + fapi2::buffer l_scomMask(0); + bool l_other_side_functional = false; + bool l_same_side_functional = false; + uint8_t l_current_pos = 0; + uint8_t l_other_side_pos = 0; + uint8_t l_same_side_pos = 0; + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_mcTarget, + l_current_pos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Calculate the peer MC in the other side and in the same side. + l_other_side_pos = (l_current_pos + MAX_MC_PER_SIDE) % MAX_MC_PER_PROC; + l_same_side_pos = ((l_current_pos / MAX_MC_SIDES_PER_PROC) * MAX_MC_PER_SIDE) + + ((l_current_pos % MAX_MC_PER_SIDE) + 1) % MAX_MC_PER_SIDE; + + FAPI_DBG("Current pos: %i, other side pos: %i, same side pos: %i", + l_current_pos, l_other_side_pos, l_same_side_pos); + + // Determine side functionality + for (auto l_mc : i_mcTargets) + { + + uint8_t l_tmp_pos = 0; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, + l_tmp_pos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // The other side + if (l_tmp_pos == l_other_side_pos) + { + l_other_side_functional = true; + } + + // the same side + if (l_tmp_pos == l_same_side_pos) + { + l_same_side_functional = true; + } + } + + l_scomData.flush<0>(); + l_scomMask.flush<0>(); + + if (!l_other_side_functional) + { + l_scomData.setBit(); + l_scomMask.setBit(); + } + + if (!l_same_side_functional) + { + l_scomData.setBit(); + l_scomMask.setBit(); + } + + FAPI_INF("Writing MCS_MCMODE0 reg 0x%.16llX: Mask 0x%.16llX , Data 0x%.16llX", + MCS_MCMODE0, l_scomMask, l_scomData); + + FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCMODE0, + l_scomData, l_scomMask), + "putScomUnderMask() returns an error (Mode0), MCS_MCMODE0 reg 0x%.16llX", + MCS_MCMODE0); + +fapi_try_exit: + FAPI_DBG("Exiting progMCMODE0"); + return fapi2::current_err; +} /// /// @brief Programming master MCS /// Writes MCS_MCSYNC reg to set the input MCS as the master. /// +/// @tparam T template parameter, passed in targets. /// @param[in] i_mcsTarget The MCS target to be programmed as master. /// /// @return FAPI2_RC_SUCCESS if success, else error code. /// -fapi2::ReturnCode progMasterMcs( - const fapi2::Target& i_mcsTarget) +template< fapi2::TargetType T> +fapi2::ReturnCode progMaster( + const fapi2::Target& i_mcTarget) { - FAPI_DBG("Entering"); + FAPI_DBG("Entering progMaster"); fapi2::ReturnCode l_rc; fapi2::buffer l_scomData(0); fapi2::buffer l_scomMask(0); @@ -96,13 +240,13 @@ fapi2::ReturnCode progMasterMcs( l_scomMask.flush<0>().setBit(); l_scomData.flush<0>(); - FAPI_TRY(fapi2::putScomUnderMask(i_mcsTarget, MCS_MCSYNC, + FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCSYNC, l_scomData, l_scomMask), "putScomUnderMask() returns an error (Sync reset), Addr 0x%.16llX", MCS_MCSYNC); // -------------------------------------------------------------- - // 2. Setup MC Sync Command Register data for master MCS + // 2. Setup MC Sync Command Register data for master MCS or MI // -------------------------------------------------------------- // Clear buffers l_scomData.flush<0>(); @@ -117,6 +261,8 @@ fapi2::ReturnCode progMasterMcs( // Setup MCSYNC_SYNC_TYPE // Set all sync types except Super Sync + // SUPER_SYNC_BIT == bit 14, supersync for Nimbus, reserved for cumulus. + // Clear it in both cases. l_scomData.setBit().clearBit(SUPER_SYNC_BIT); l_scomMask.setBit(); // -------------------------------------------------------------- - // 3. Write to MC Sync Command Register of master MCS + // 3. Write to MC Sync Command Register of master MCS or MI // -------------------------------------------------------------- // Write to MCSYNC reg FAPI_INF("Writing MCS_MCSYNC reg 0x%.16llX: Mask 0x%.16llX , Data 0x%.16llX", MCS_MCSYNC, l_scomMask, l_scomData); - FAPI_TRY(fapi2::putScomUnderMask(i_mcsTarget, MCS_MCSYNC, + FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCSYNC, l_scomData, l_scomMask), "putScomUnderMask() returns an error (Sync), MCS_MCSYNC reg 0x%.16llX", MCS_MCSYNC); @@ -141,31 +287,27 @@ fapi2::ReturnCode progMasterMcs( // Note: No need to read Sync replay count and retry in P9. fapi_try_exit: - FAPI_DBG("Exiting"); + FAPI_DBG("Exiting progMaster"); return fapi2::current_err; } - /// /// @brief Perform throttle sync on the Memory Controllers /// /// @tparam T template parameter, passed in targets. -/// @param[in] i_mcTargets Vector of reference of MC targets (MCS or MI) +/// @param[in] i_mcTargets Vector of reference of MC targets (MCS or MI) +/// @param[in] i_HW397255_enabled Workaround for HW397255 /// /// @return FAPI2_RC_SUCCESS if success, else error code. /// template< fapi2::TargetType T> fapi2::ReturnCode throttleSync( - const std::vector< fapi2::Target >& i_mcTargets); - -/// TARGET_TYPE_MCS -template<> -fapi2::ReturnCode throttleSync( - const std::vector< fapi2::Target >& i_mcTargets) + const std::vector< fapi2::Target >& i_mcTargets, + fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type i_HW397255_enabled) { FAPI_DBG("Entering"); fapi2::ReturnCode l_rc; - mcSideInfo_t l_mcSide[MAX_MC_SIDES_PER_PROC]; + mcSideInfo_t l_mcSide[MAX_MC_SIDES_PER_PROC]; uint8_t l_sideNum = 0; uint8_t l_pos = 0; uint8_t l_numMasterProgrammed = 0; @@ -173,25 +315,24 @@ fapi2::ReturnCode throttleSync( // Initialization for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++) { - l_mcSide[l_sideNum].masterMcsFound = false; + l_mcSide[l_sideNum].masterMcFound = false; } // --------------------------------------------------------------------- - // 1. Pick the first MCS with DIMMS as potential master + // 1. Pick the first MCS/MI with DIMMS as potential master // for both MC sides (MC01/MC23) // --------------------------------------------------------------------- - for (auto l_mcs : i_mcTargets) + for (auto l_mc : i_mcTargets) { - std::vector< fapi2::Target > l_dimms = - mss::find_targets(l_mcs); + uint8_t l_num_dimms = findNumDimms(l_mc); - if (l_dimms.size() > 0) + if (l_num_dimms > 0) { - // This MCS has DIMMs attached, find out which MC side it + // This MCS or MI has DIMMs attached, find out which MC side it // belongs to: // l_sideNum = 0 --> MC01 // 1 --> MC23 - FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mcs, + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, l_pos), "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", (uint64_t)fapi2::current_err); @@ -199,52 +340,51 @@ fapi2::ReturnCode throttleSync( FAPI_INF("MCS %u has DIMMs", l_pos); - // If there's no master MCS marked for this side yet, mark + // If there's no master MCS or MI marked for this side yet, mark // this MCS as master - if (l_mcSide[l_sideNum].masterMcsFound == false) + if (l_mcSide[l_sideNum].masterMcFound == false) { FAPI_INF("Mark MCS %u as master for MC side %u", l_pos, l_sideNum); - l_mcSide[l_sideNum].masterMcsFound = true; - l_mcSide[l_sideNum].masterMcs = l_mcs; + l_mcSide[l_sideNum].masterMcFound = true; + l_mcSide[l_sideNum].masterMc = l_mc; } } + + // Program the MCMODE0 if HW397255 is not enabled which means we + // should have a chip with Nimbus DD2+ or Cumulus. + if (i_HW397255_enabled == 0) + { + progMCMODE0(l_mc, i_mcTargets); + } } // -------------------------------------------------------------- - // 2. Program the master MCS + // 2. Program the master MCS or MI // -------------------------------------------------------------- for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++) { - // If there is a potential master MCS found for this side - if (l_mcSide[l_sideNum].masterMcsFound == true) + // If there is a potential master MCS or MI found for this side + if (l_mcSide[l_sideNum].masterMcFound == true) { - // No master MCS programmed for either side yet, - // go ahead and program this MCS as master. + // No master MCS or MI programmed for either side yet, + // go ahead and program this MCS or MI as master. if (l_numMasterProgrammed == 0) { - FAPI_TRY(progMasterMcs(l_mcSide[l_sideNum].masterMcs), - "programMasterMcs() returns error" + FAPI_TRY(progMaster(l_mcSide[l_sideNum].masterMc), + "programMaster() returns error" "NumMasterProgrammed %d, l_rc 0x%.8X", l_numMasterProgrammed, (uint64_t)fapi2::current_err); l_numMasterProgrammed++; } else { - // A master MCS is already programmed on MC side 0 (MC01). // HW397255 requires to also program a master MCS on MC23 if // it has DIMMs. - fapi2::Target l_chip = - l_mcSide[l_sideNum].masterMcs.getParent(); - fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type l_HW397255_enabled; - FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_HW397255, - l_chip, l_HW397255_enabled), - "Error getting the ATTR_CHIP_EC_FEATURE_HW397255"); - - if (l_HW397255_enabled) + if (i_HW397255_enabled == 1) { - FAPI_TRY(progMasterMcs(l_mcSide[l_sideNum].masterMcs), - "programMasterMcs() returns error" + FAPI_TRY(progMaster(l_mcSide[l_sideNum].masterMc), + "programMaster() returns error" "NumMasterProgrammed %d, l_rc 0x%.8X", l_numMasterProgrammed, (uint64_t)fapi2::current_err); } @@ -257,20 +397,6 @@ fapi_try_exit: return fapi2::current_err; } -/// TARGET_TYPE_MI -template<> -fapi2::ReturnCode throttleSync( - const std::vector< fapi2::Target >& i_mcTargets) -{ - FAPI_DBG("Entering"); - fapi2::ReturnCode l_rc; - - // Note: Add code for Cumulus - - FAPI_DBG("Exiting"); - return fapi2::current_err; -} - extern "C" { @@ -291,10 +417,19 @@ extern "C" auto l_mcsChiplets = i_target.getChildren(); auto l_miChiplets = i_target.getChildren(); + // HW397255 requires to also program a master MCS on MC23 if + // it has DIMMs. + // This should only be enabled for Nimbus DD1, disabled for Nimbus DD2 + // and Cumulus. + fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type l_HW397255_enabled; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_HW397255, + i_target, l_HW397255_enabled), + "Error getting the ATTR_CHIP_EC_FEATURE_HW397255"); + // Get the functional MCS on this proc if (l_mcsChiplets.size() > 0) { - FAPI_TRY(throttleSync(l_mcsChiplets), + FAPI_TRY(throttleSync(l_mcsChiplets, l_HW397255_enabled), "throttleSync() returns error l_rc 0x%.8X", (uint64_t)fapi2::current_err); } @@ -302,7 +437,7 @@ extern "C" // Cumulus if (l_miChiplets.size() > 0) { - FAPI_TRY(throttleSync(l_miChiplets), + FAPI_TRY(throttleSync(l_miChiplets, l_HW397255_enabled), "throttleSync() returns error l_rc 0x%.8X", (uint64_t)fapi2::current_err); } -- cgit v1.2.1