From 264bea8053f263073fee6988e860944b4c427637 Mon Sep 17 00:00:00 2001 From: Mark Pizzutillo Date: Wed, 26 Jun 2019 17:50:36 -0500 Subject: Add p9a_throttle_sync procedure Change-Id: Ia89d521993387b4467b334f29a9be3f93bda8f75 Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79579 Tested-by: Jenkins Server Tested-by: FSP CI Jenkins Reviewed-by: Louis Stermole Reviewed-by: Michael D. Pardeik Tested-by: HWSV CI Tested-by: Hostboot CI Reviewed-by: Jennifer A Stofer Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79583 Tested-by: Jenkins OP Build CI Tested-by: Jenkins OP HW Reviewed-by: Christian R. Geddes --- .../p9/procedures/hwp/nest/p9a_throttle_sync.C | 322 +++++++++++++++++++++ .../p9/procedures/hwp/nest/p9a_throttle_sync.H | 54 ++++ .../p9/procedures/hwp/nest/p9a_throttle_sync.mk | 6 + .../hwp/memory/lib/shared/axone_consts.H | 4 + src/import/generic/memory/lib/utils/find.H | 45 +++ 5 files changed, 431 insertions(+) (limited to 'src') diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.C b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.C index f5cbb3787..c288c1fb2 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.C +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.C @@ -22,3 +22,325 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ +/// ---------------------------------------------------------------------------- +/// @file p9_throttle_sync.C +/// +/// @brief Perform p9_throttle_sync HWP +/// +/// The purpose of this procedure is to triggers sync command from a 'master' +/// MC to other MCs that have attached memory in a processor. +/// +/// ---------------------------------------------------------------------------- +/// *HWP HWP Owner : Joe McGill +/// *HWP HWP Backup : Mark Pizzutillo +/// *HWP FW Owner : Thi Tran +/// *HWP Team : Nest +/// *HWP Level : 3 +/// *HWP Consumed by : HB +/// ---------------------------------------------------------------------------- + +//------------------------------------------------------------------------------ +// Includes +//------------------------------------------------------------------------------ +#include +#include +#include +#include +#include + +/// +/// @brief Program MCMODE0 based on the functional targets +/// +/// @param[in] i_mi_target The MC target to be programmed +/// @param[in] i_mi_targets Other MC targets. +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode prog_MCMODE0( + const fapi2::Target& i_mc_target, + const std::vector< fapi2::Target >& i_mc_targets) +{ + FAPI_DBG("Entering prog_MCMODE0 on target %s", mss::c_str(i_mc_target)); + // -------------------------------------------------------------- + // Setup MCMODE0 for disabling MC SYNC to other-side and same-side + // partner unit. + // BIT27: set if other-side MC is non-functional, 0<->2, 1<->3 + // BIT28: set if same-side MC is non-functional, 0<->1, 2<->3 + // -------------------------------------------------------------- + fapi2::buffer l_scomData(0); + fapi2::buffer l_scomMask(0); + bool l_other_side_functional = false; + bool l_same_side_functional = false; + uint8_t l_current_pos = 0; + uint8_t l_other_side_pos = 0; + uint8_t l_same_side_pos = 0; + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_mc_target, l_current_pos), + "Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(i_mc_target)); + + // Calculate the peer MC in the other side and in the same side. + l_other_side_pos = (l_current_pos + MAX_MC_PER_SIDE) % MAX_MC_PER_PROC; + l_same_side_pos = ((l_current_pos / MAX_MC_SIDES_PER_PROC) * MAX_MC_PER_SIDE) + + ((l_current_pos % MAX_MC_PER_SIDE) + 1) % MAX_MC_PER_SIDE; + + FAPI_DBG("Current pos: %i, other side pos: %i, same side pos: %i", + l_current_pos, l_other_side_pos, l_same_side_pos); + + // Determine side functionality + for (const auto& l_mc : i_mc_targets) + { + uint8_t l_tmp_pos = 0; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, l_tmp_pos), + "Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(l_mc)); + + // The other side + if (l_tmp_pos == l_other_side_pos) + { + l_other_side_functional = true; + } + + // The same side + if (l_tmp_pos == l_same_side_pos) + { + l_same_side_functional = true; + } + } + + l_scomData.flush<0>(); + l_scomMask.flush<0>(); + + if (!l_other_side_functional) + { + l_scomData.setBit(); + l_scomMask.setBit(); + } + else + { + l_scomData.clearBit(); + l_scomMask.setBit(); + } + + if (!l_same_side_functional) + { + l_scomData.setBit(); + l_scomMask.setBit(); + } + else + { + l_scomData.clearBit(); + l_scomMask.setBit(); + } + + FAPI_DBG("Writing MCS_MCMODE0 reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX", + P9A_MI_MCMODE0, l_scomMask, l_scomData); + + FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCMODE0, l_scomData, l_scomMask), + "putScomUnderMask() returns an error, P9A_MI_MCMODE0 reg 0x%016llX", P9A_MI_MCMODE0); + +fapi_try_exit: + FAPI_DBG("Exiting prog_MCMODE0"); + return fapi2::current_err; +} + +/// +/// @brief Programming master MCS +/// Writes P9A_MI_MCSYNC reg to set the input MCS as the master. +/// +/// @param[in] i_mcsTarget The MCS target to be programmed as master. +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode prog_master(const fapi2::Target& i_mc_target) +{ + FAPI_DBG("Entering progMaster with target %s", mss::c_str(i_mc_target)); + fapi2::buffer l_scomData(0); + fapi2::buffer l_scomMask(0); + + // ------------------------------------------------------------------- + // 1. Reset sync command + // ------------------------------------------------------------------- + + // Clear GO bit + l_scomMask.flush<0>().setBit(); + l_scomData.flush<0>(); + FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCSYNC, l_scomData, l_scomMask), + "putScomUnderMask() returns an error (Reset), P9A_MI_MCSYNC reg 0x%016llX", P9A_MI_MCSYNC); + + // -------------------------------------------------------------- + // 2. Setup MC Sync Command Register data for master MCS or MI + // -------------------------------------------------------------- + + // Clear buffers + l_scomData.flush<0>(); + l_scomMask.flush<0>(); + + // Force bit set in case cleared from last procedure run + l_scomData.setBit(); + l_scomMask.setBit(); + + // Iterate through OCMBs to make sure refresh SYNC bit is set + for (const auto& l_ocmb : mss::find_targets(i_mc_target)) + { + FAPI_DBG("Writing EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX: Data 0x%016llX Mask 0x%016llX", + EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask); + + FAPI_TRY(fapi2::putScomUnderMask(l_ocmb, EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask), + "Error writing to REG 0x%016llX of %s", EXPLR_SRQ_MBA_SYNCCNTLQ, mss::c_str(l_ocmb)); + } + + // Clear buffers + l_scomData.flush<0>(); + l_scomMask.flush<0>(); + + // Setup MCSYNC_CHANNEL_SELECT + // Set ALL channels with or without DIMMs (bits 0:7) + l_scomData.setBit(); + l_scomMask.setBit(); + + // Setup MCSYNC_SYNC_TYPE for SYNC ALL + l_scomData.setBit(); + l_scomMask.setBit(); + + // Setup SYNC_GO (bit 16 is now used for both channels) + l_scomMask.setBit(); + l_scomData.setBit(); + + // -------------------------------------------------------------- + // 3. Write to MC Sync Command Register of master MCS or MI + // -------------------------------------------------------------- + // Write to MCSYNC reg + FAPI_DBG("Writing P9A_MI_MCSYNC reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX", + P9A_MI_MCSYNC, l_scomMask, l_scomData); + + FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCSYNC, l_scomData, l_scomMask), + "putScomUnderMask() returns an error (Sync), P9A_MI_MCSYNC reg 0x%016llX", P9A_MI_MCSYNC); + + // Note: No need to read Sync replay count and retry in P9. + + // -------------------------------------------------------------- + // 4. Clear refresh sync bit + // -------------------------------------------------------------- + l_scomData.flush<0>(); + l_scomMask.flush<0>().setBit(); + + // Iterate through OCMBs to clear refresh sync bit + for (const auto& l_ocmb : mss::find_targets(i_mc_target)) + { + FAPI_DBG("Writing EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX on %s", + EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomMask, l_scomData, mss::c_str(l_ocmb)); + + FAPI_TRY(fapi2::putScomUnderMask(l_ocmb, EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask), + "putScomUnderMask() returns an error (Sync), EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX", + EXPLR_SRQ_MBA_SYNCCNTLQ); + } + +fapi_try_exit: + FAPI_DBG("Exiting progMaster"); + return fapi2::current_err; +} + +/// +/// @brief Perform throttle sync on the Memory Controllers +/// +/// @param[in] i_mc_targets vector of reference of MC targets (MCS or MI) +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode throttle_sync( + const std::vector< fapi2::Target >& i_mc_targets) +{ + mc_side_info_t l_mcSide[MAX_MC_SIDES_PER_PROC]; + uint8_t l_sideNum = 0; + uint8_t l_pos = 0; + uint8_t l_numMasterProgrammed = 0; + + // Initialization + for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++) + { + l_mcSide[l_sideNum].master_mc_found = false; + } + + // --------------------------------------------------------------------- + // 1. Pick the first MCS/MI with DIMMS as potential master + // for both MC sides (MC01/MC23) + // --------------------------------------------------------------------- + for (const auto& l_mc : i_mc_targets) + { + uint8_t l_num_dimms = mss::find_targets(l_mc).size(); + + if (l_num_dimms > 0) + { + // This MCS or MI has DIMMs attached, find out which MC side it + // belongs to: + // l_sideNum = 0 --> MC01 + // 1 --> MC23 + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, l_pos), + "Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(l_mc)); + l_sideNum = l_pos / MAX_MC_SIDES_PER_PROC; + + FAPI_INF("MCS %u has DIMMs", l_pos); + + // If there's no master MCS or MI marked for this side yet, mark + // this MCS as master + if (l_mcSide[l_sideNum].master_mc_found == false) + { + FAPI_INF("Mark MCS %u as master for MC side %u", + l_pos, l_sideNum); + l_mcSide[l_sideNum].master_mc_found = true; + l_mcSide[l_sideNum].master_mc = l_mc; + } + } + + prog_MCMODE0(l_mc, i_mc_targets); + } + + // -------------------------------------------------------------- + // 2. Program the master MI + // -------------------------------------------------------------- + for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++) + { + // If there is a potential master MI found for this side + if (l_mcSide[l_sideNum].master_mc_found == true) + { + // No master MI programmed for either side yet, + // go ahead and program this MI as master. + if (l_numMasterProgrammed == 0) + { + FAPI_TRY(prog_master(l_mcSide[l_sideNum].master_mc), + "programMaster() returns error on %s", mss::c_str(l_mcSide[l_sideNum].master_mc)); + l_numMasterProgrammed++; + } + } + } + +fapi_try_exit: + FAPI_DBG("Exiting"); + return fapi2::current_err; +} + +extern "C" +{ + /// + /// @brief p9a_throttle_sync procedure + /// + /// @param[in] i_target TARGET_TYPE_PROC_CHIP target + /// @return FAPI2_RC_SUCCESS if success, else error code. + /// + fapi2::ReturnCode p9a_throttle_sync( + const fapi2::Target& i_target) + { + FAPI_DBG("Executing p9a_throttle_sync on %s", mss::c_str(i_target)); + + const auto l_miChiplets = i_target.getChildren(); + + if (l_miChiplets.size() > 0) + { + FAPI_TRY(throttle_sync(l_miChiplets), "Error calling throttle_sync() with vector of MI Chiplets"); + } + + fapi_try_exit: + FAPI_DBG("Exiting"); + return fapi2::current_err; + } + +} // extern "C" diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.H b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.H index ac845a974..c87312b23 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.H +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.H @@ -22,3 +22,57 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ +/// ---------------------------------------------------------------------------- +/// @file p9a_throttle_sync.H +/// +/// @brief p9a_throttle_sync HWP +/// +/// The purpose of this procedure is to triggers sync command from a 'master' +/// MC to other MCs that have attached memory in a processor. +/// +/// ---------------------------------------------------------------------------- +/// *HWP HWP Owner : Joe McGill +/// *HWP HWP Backup : Mark Pizzutillo +/// *HWP FW Owner : Thi Tran +/// *HWP Team : Nest +/// *HWP Level : 3 +/// *HWP Consumed by : HB +/// ---------------------------------------------------------------------------- +#ifndef _P9A_THROTTLE_SYNC_H_ +#define _P9A_THROTTLE_SYNC_H_ + +//------------------------------------------------------------------------------ +// Includes +//------------------------------------------------------------------------------ +#include +#include +#include +#include +#include +#include + +// Function pointer typedef definition for HWP call support +typedef fapi2::ReturnCode (*p9a_throttle_sync_FP_t)(const fapi2::Target&); + +/// +/// @brief Structure that holds the potential master MI for a MC side (MC01/MC23) +/// +struct mc_side_info_t +{ + bool master_mc_found = false; + fapi2::Target master_mc; // Master MC for this MC side +}; + +extern "C" +{ + /// + /// @brief p9a_throttle_sync procedure + /// + /// @param[in] i_target TARGET_TYPE_PROC_CHIP target + /// @return FAPI2_RC_SUCCESS if success, else error code. + /// + fapi2::ReturnCode p9a_throttle_sync(const fapi2::Target& i_target); + +} // extern "C" + +#endif // _P9A_THROTTLE_SYNC_H_ diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.mk b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.mk index 226db9ed0..cf235b53d 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.mk +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.mk @@ -22,3 +22,9 @@ # permissions and limitations under the License. # # IBM_PROLOG_END_TAG + +PROCEDURE=p9a_throttle_sync +$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9a/procedures/hwp/memory/) + +$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)) +$(call BUILD_PROCEDURE) diff --git a/src/import/chips/p9a/procedures/hwp/memory/lib/shared/axone_consts.H b/src/import/chips/p9a/procedures/hwp/memory/lib/shared/axone_consts.H index c2ef9aee8..dcca78fa0 100644 --- a/src/import/chips/p9a/procedures/hwp/memory/lib/shared/axone_consts.H +++ b/src/import/chips/p9a/procedures/hwp/memory/lib/shared/axone_consts.H @@ -39,4 +39,8 @@ #include +const uint8_t MAX_MC_SIDES_PER_PROC = 2; // MC01, MC23 +const uint8_t MAX_MC_PER_PROC = 4; // MC0, MC1, MC2, MC3 +const uint8_t MAX_MC_PER_SIDE = 2; // MC0, MC1 or MC2, MC3 + #endif diff --git a/src/import/generic/memory/lib/utils/find.H b/src/import/generic/memory/lib/utils/find.H index ee2a5b8c1..3e67e765a 100644 --- a/src/import/generic/memory/lib/utils/find.H +++ b/src/import/generic/memory/lib/utils/find.H @@ -232,6 +232,51 @@ find_targets( const fapi2::Target& i_target, return l_ocmbs; } +/// +/// @brief find all the DIMMS connected to an MI +/// @param[in] i_target a fapi2::Target MI +/// @return a vector of fapi2::TARGET_TYPE_DIMM +/// +template<> +inline std::vector< fapi2::Target > +find_targets( const fapi2::Target& i_target, + fapi2::TargetState i_state ) +{ + std::vector< fapi2::Target > l_dimms; + + for (const auto& l_omi : i_target.getChildren(i_state)) + { + for (const auto& l_ocmb : l_omi.getChildren(i_state)) + { + auto l_these_dimms( l_ocmb.getChildren(i_state) ); + l_dimms.insert(l_dimms.end(), l_these_dimms.begin(), l_these_dimms.end()); + } + } + + return l_dimms; +} + +/// +/// @brief find all the OCMB_CHIPs connected to an MI +/// @param[in] i_target a fapi2::Target MI +/// @return a vector of fapi2::TARGET_TYPE_OCMB_CHIP +/// +template<> +inline std::vector< fapi2::Target > +find_targets( const fapi2::Target& i_target, + fapi2::TargetState i_state ) +{ + std::vector< fapi2::Target > l_ocmbs; + + for (const auto& l_omi : i_target.getChildren(i_state)) + { + auto l_these_ocmbs( l_omi.getChildren(i_state) ); + l_ocmbs.insert(l_ocmbs.end(), l_these_ocmbs.begin(), l_these_ocmbs.end()); + } + + return l_ocmbs; +} + /// /// @brief find all the MEM_PORTs connected to a PROC_CHIP /// @param[in] i_target a fapi2::Target PROC_CHIP -- cgit v1.2.1