diff options
| author | Ben Gass <bgass@us.ibm.com> | 2018-10-19 07:32:27 -0500 |
|---|---|---|
| committer | Christian R. Geddes <crgeddes@us.ibm.com> | 2019-04-04 09:25:38 -0500 |
| commit | b823dd50837a9f05cf69dc75d5942319f537024d (patch) | |
| tree | 7414d5b461004c3d475ba2cb0ed25bd11c5b71c6 /src/import/chips/p9/procedures/hwp | |
| parent | e5622fb032dc8b23627d8ca06d82a10b612d5435 (diff) | |
| download | talos-hostboot-b823dd50837a9f05cf69dc75d5942319f537024d.tar.gz talos-hostboot-b823dd50837a9f05cf69dc75d5942319f537024d.zip | |
Update p9_mss_eff_grouping for Axone support
Add p9a_omi_setup_bars procedure
Add eclipse project files to .gitignore
Change-Id: Ia18cd213ac8b3682e5718b3c631dad631b97170f
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/67755
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/67764
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: Christian R. Geddes <crgeddes@us.ibm.com>
Diffstat (limited to 'src/import/chips/p9/procedures/hwp')
15 files changed, 2193 insertions, 180 deletions
diff --git a/src/import/chips/p9/procedures/hwp/initfiles/p9_cxa_scom.C b/src/import/chips/p9/procedures/hwp/initfiles/p9_cxa_scom.C index 29d6e2b1c..affeb906f 100644 --- a/src/import/chips/p9/procedures/hwp/initfiles/p9_cxa_scom.C +++ b/src/import/chips/p9/procedures/hwp/initfiles/p9_cxa_scom.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2016,2018 */ +/* Contributors Listed Below - COPYRIGHT 2016,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -154,7 +154,7 @@ fapi2::ReturnCode p9_cxa_scom(const fapi2::Target<fapi2::TARGET_TYPE_CAPP>& TGT0 if (((l_chip_id == 0x5) && (l_chip_ec == 0x20)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x21)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x22)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x23)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x10)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x11)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x12)) || ((l_chip_id == 0x6) - && (l_chip_ec == 0x13)) ) + && (l_chip_ec == 0x13)) || ((l_chip_id == 0x7) && (l_chip_ec == 0x10)) ) { l_scom_buffer.insert<21, 4, 60, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID ); l_scom_buffer.insert<25, 3, 61, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID ); diff --git a/src/import/chips/p9/procedures/hwp/initfiles/p9_int_scom.C b/src/import/chips/p9/procedures/hwp/initfiles/p9_int_scom.C index da4f761cb..8951137fa 100644 --- a/src/import/chips/p9/procedures/hwp/initfiles/p9_int_scom.C +++ b/src/import/chips/p9/procedures/hwp/initfiles/p9_int_scom.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2016,2018 */ +/* Contributors Listed Below - COPYRIGHT 2016,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -97,7 +97,7 @@ fapi2::ReturnCode p9_int_scom(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& if (((l_chip_id == 0x5) && (l_chip_ec == 0x20)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x21)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x22)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x23)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x10)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x11)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x12)) || ((l_chip_id == 0x6) - && (l_chip_ec == 0x13)) ) + && (l_chip_ec == 0x13)) || ((l_chip_id == 0x7) && (l_chip_ec == 0x10)) ) { l_scom_buffer.insert<5, 4, 60, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID ); l_scom_buffer.insert<9, 3, 61, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID ); diff --git a/src/import/chips/p9/procedures/hwp/initfiles/p9_nx_scom.C b/src/import/chips/p9/procedures/hwp/initfiles/p9_nx_scom.C index e48166c54..6ac32f566 100644 --- a/src/import/chips/p9/procedures/hwp/initfiles/p9_nx_scom.C +++ b/src/import/chips/p9/procedures/hwp/initfiles/p9_nx_scom.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2016,2018 */ +/* Contributors Listed Below - COPYRIGHT 2016,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -662,7 +662,7 @@ fapi2::ReturnCode p9_nx_scom(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& if (((l_chip_id == 0x5) && (l_chip_ec == 0x20)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x21)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x22)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x23)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x10)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x11)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x12)) || ((l_chip_id == 0x6) - && (l_chip_ec == 0x13)) ) + && (l_chip_ec == 0x13)) || ((l_chip_id == 0x7) && (l_chip_ec == 0x10)) ) { l_scom_buffer.insert<56, 4, 60, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID ); l_scom_buffer.insert<60, 3, 61, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID ); diff --git a/src/import/chips/p9/procedures/hwp/initfiles/p9_vas_scom.C b/src/import/chips/p9/procedures/hwp/initfiles/p9_vas_scom.C index 44cd049f9..69b80637c 100644 --- a/src/import/chips/p9/procedures/hwp/initfiles/p9_vas_scom.C +++ b/src/import/chips/p9/procedures/hwp/initfiles/p9_vas_scom.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2016,2018 */ +/* Contributors Listed Below - COPYRIGHT 2016,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -126,7 +126,7 @@ fapi2::ReturnCode p9_vas_scom(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& if (((l_chip_id == 0x5) && (l_chip_ec == 0x20)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x21)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x22)) || ((l_chip_id == 0x5) && (l_chip_ec == 0x23)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x10)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x11)) || ((l_chip_id == 0x6) && (l_chip_ec == 0x12)) || ((l_chip_id == 0x6) - && (l_chip_ec == 0x13)) ) + && (l_chip_ec == 0x13)) || ((l_chip_id == 0x7) && (l_chip_ec == 0x10)) ) { l_scom_buffer.insert<0, 4, 60, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID ); l_scom_buffer.insert<4, 3, 61, uint64_t>(l_TGT1_ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID ); diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.C b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.C index 231c83426..342e32247 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.C +++ b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.C @@ -90,6 +90,16 @@ enum GroupAllowed GROUP_8, }; +// ----------------------- +// Used to indicate which subchannels of an OMI channel are used +enum OMISubChannelConfig +{ + NONE = 0b00000000, // Neither sub channel is enabled + A = 0b10000000, // Sub-channel A is used + B = 0b01000000, // Sub-channel B is used + BOTH = A | B, // Both are enabled (mirroring is allowed) +}; + // matrix for port-pair based deconfiguration combinations const uint8_t MAX_MBA_PERMUTATIONS = 9; const uint8_t MBA_PERMUTATIONS[MAX_MBA_PERMUTATIONS][NUM_PORTS_PER_PAIR][NUM_MBA_PER_MEMBUF] = @@ -160,6 +170,8 @@ struct EffGroupingSysAttrs /// fapi2::ReturnCode getAttrs(); + void updateGroupsAllowed(bool i_is_axone); + // Public data uint8_t iv_selectiveMode = 0; // ATTR_MEM_MIRROR_PLACEMENT_POLICY uint8_t iv_hwMirrorEnabled = 0; // ATTR_MRW_HW_MIRRORING_ENABLE @@ -167,8 +179,42 @@ struct EffGroupingSysAttrs uint8_t iv_smfSupported = 0; // ATTR_CHIP_EC_FEATURE_SMF_SUPPORTED uint8_t iv_smfConfig = 0; // ATTR_SMF_CONFIG uint8_t iv_smfEnabled = 0; // ATTR_SMF_ENABLED + fapi2::ATTR_CHIP_EC_FEATURE_HW423589_OPTION2_Type iv_hw423589_option2; }; +void EffGroupingSysAttrs::updateGroupsAllowed(bool i_is_axone) +{ + // derate requested interleave options based on hardware workarounds/ + // restrictions or mirroring requirements + + // 1. Disallow 6-way/3-way interleave to prevent overflow of 512 GB + // footprint with 7 of 8 max-sized DIMMs present + if (iv_hw423589_option2) + { + FAPI_INF("Groups of 6 & 3 are not allowed with HW423589 option2"); + iv_groupsAllowed &= ~GROUP_6; + iv_groupsAllowed &= ~GROUP_3; + } + + // 2. HW423110 - Disallow 6-way interleave if mirroring is desired + if (iv_hwMirrorEnabled != fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_FALSE && !i_is_axone) + { + FAPI_INF("Group of 6 is not allowed with Mirroring"); + iv_groupsAllowed &= ~GROUP_6; + } + + // 3. SW433435 - Disallow 3-way/1-way interleave if all channels are required + // to be mirrored (unless Axone). 2-way cross port interleaving is also prohibited + // in this case, but is handled in grouping_group2PortsPerGroup() + if (iv_hwMirrorEnabled == fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_TRUE && !i_is_axone) + { + FAPI_INF("Groups of 3 & 1 are not allowed with Mirroring Required"); + iv_groupsAllowed &= ~GROUP_3; + iv_groupsAllowed &= ~GROUP_1; + } +} + + // See doxygen in struct definition. fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() { @@ -180,7 +226,6 @@ fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() uint8_t l_addr_extension_chip_id = 0; uint64_t l_max_interleave_group_size; fapi2::ATTR_CHIP_EC_FEATURE_EXTENDED_ADDRESSING_MODE_Type l_extended_addressing_mode = 0; - fapi2::ATTR_CHIP_EC_FEATURE_HW423589_OPTION2_Type l_hw423589_option2 = 0; auto l_targets = FAPI_SYSTEM.getChildren<fapi2::TARGET_TYPE_PROC_CHIP>(); @@ -193,12 +238,12 @@ fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_HW423589_OPTION2, l_targets.front(), - l_hw423589_option2), + iv_hw423589_option2), "Error from FAPI_ATTR_GET (ATTR_CHIP_EC_FEATURE_HW423589_OPTION2)"); FAPI_DBG("Extended addressing supported: %d, HW423589 option2: %d", l_extended_addressing_mode, - l_hw423589_option2); + iv_hw423589_option2); FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_SMF_CONFIG, FAPI_SYSTEM, iv_smfConfig), "Error from FAPI_ATTR_GET (ATTR_SMF_CONFIG)"); @@ -211,7 +256,7 @@ fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() if (l_extended_addressing_mode) { - if (l_hw423589_option2) + if (iv_hw423589_option2) { l_addr_extension_group_id = CHIP_ADDRESS_EXTENSION_GROUP_ID_MASK_HW423589_OPTION2; l_addr_extension_chip_id = CHIP_ADDRESS_EXTENSION_CHIP_ID_MASK_HW423589_OPTION2; @@ -236,7 +281,7 @@ fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() "Error from FAPI_ATTR_SET (ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID)"); } - if (l_hw423589_option2) + if (iv_hw423589_option2) { // restrict max size for MCD issue l_max_interleave_group_size = MAX_INTERLEAVE_GROUP_SIZE_HW423589_OPTION2; @@ -277,35 +322,6 @@ fapi2::ReturnCode EffGroupingSysAttrs::getAttrs() "Error getting ATTR_MSS_INTERLEAVE_ENABLE, l_rc 0x%.8X", (uint64_t)fapi2::current_err); - // derate requested interleave options based on hardware workarounds/ - // restrictions or mirroring requirements - - // 1. Disallow 6-way/3-way interleave to prevent overflow of 512 GB - // footprint with 7 of 8 max-sized DIMMs present - if (l_hw423589_option2) - { - FAPI_INF("Groups of 6 & 3 are not allowed with HW423589 option2"); - iv_groupsAllowed &= ~GROUP_6; - iv_groupsAllowed &= ~GROUP_3; - } - - // 2. HW423110 - Disallow 6-way interleave if mirroring is desired - if (iv_hwMirrorEnabled != fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_FALSE) - { - FAPI_INF("Group of 6 is not allowed with Mirroring"); - iv_groupsAllowed &= ~GROUP_6; - } - - // 3. SW433435 - Disallow 3-way/1-way interleave if all channels are required - // to be mirrored. 2-way cross port interleaving is also prohibited - // in this case, but is handled in grouping_group2PortsPerGroup() - if (iv_hwMirrorEnabled == fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_TRUE) - { - FAPI_INF("Groups of 3 & 1 are not allowed with Mirroring Required"); - iv_groupsAllowed &= ~GROUP_3; - iv_groupsAllowed &= ~GROUP_1; - } - // Display attribute values FAPI_INF("EffGroupingSysAttrs: "); FAPI_INF(" ATTR_MEM_MIRROR_PLACEMENT_POLICY 0x%.8X", iv_selectiveMode); @@ -631,6 +647,115 @@ fapi_try_exit: ///---------------------------------------------------------------------------- +/// struct EffGroupingMccAttrs +///---------------------------------------------------------------------------- +/// +/// @struct EffGroupingMccAttrs +/// +/// Contains attributes for an MCC Chiplet (Axone only) +/// +struct EffGroupingMccAttrs +{ + /// + /// @brief Getting attribute of a MCC chiplet + /// + /// Function that reads the MCC target attributes and load their + /// values into the struct. + /// + /// @param[in] i_target Reference to MCC chiplet target + /// + /// @return FAPI2_RC_SUCCESS if success, else error code. + /// + fapi2::ReturnCode getAttrs( + const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_target); + + // Unit Position + uint8_t iv_unitPos = 0; + + // Total Dimm size behind this MCC + uint64_t iv_dimmSize = 0; + + // The ocmbs associated with this MCC + // (for deconfiguring if cannot group) + std::vector<fapi2::Target<fapi2::TARGET_TYPE_OCMB_CHIP>> iv_ocmbs; + std::vector<uint8_t> iv_omi_pos; +}; + +// See doxygen in struct definition. +fapi2::ReturnCode EffGroupingMccAttrs::getAttrs( + const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_target) +{ + FAPI_DBG("Entering EffGroupingMccAttrs::getAttrs"); + + uint8_t l_omi_pos = 0; + uint64_t l_min_size = 0; + uint64_t l_ocmb_size = 0; + + // Get the ocmbs attached to this MCC + auto l_omis = i_target.getChildren<fapi2::TARGET_TYPE_OMI>(); + FAPI_DBG("Found %d omi children for MCC", l_omis.size()); + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_target, iv_unitPos), + "Error getting OMI ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // There could be up to 2 OMI/OCMB's per channel + for (auto l_omi : l_omis) + { + // Get the OMI unit position - this should match the OCMB position. + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_omi, l_omi_pos), + "Error getting OMI ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Get attached ocmb + auto l_ocmbs = l_omi.getChildren<fapi2::TARGET_TYPE_OCMB_CHIP>(); + FAPI_DBG("Found %d OCMBs attached", l_ocmbs.size()); + + if (l_ocmbs.size() > 0) + { + // Get the amount of memory behind this OCMB + FAPI_TRY(mss::eff_memory_size<mss::mc_type::EXPLORER>(l_ocmbs[0], l_ocmb_size), + "Error returned from eff_memory_size - ocmb, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + FAPI_DBG("OMI size: %llx", l_ocmb_size); + + if (l_ocmb_size > 0) + { + if (l_min_size != 0) + { + if (l_min_size != l_ocmb_size) + { + l_min_size = (l_min_size > l_ocmb_size) ? l_ocmb_size : l_min_size; + FAPI_DBG("Sub-channels for MCC %d have different size. Limiting to the smallest: %lld", + iv_unitPos, l_min_size) + } + } + else + { + l_min_size = l_ocmb_size; + } + + iv_ocmbs.push_back(l_ocmbs.front()); + iv_omi_pos.push_back(l_omi_pos); + } + } + } + + iv_dimmSize = (iv_ocmbs.size() * l_min_size); + + // Display this OMI's attribute info + FAPI_INF("EffGroupingMccAttrs::getAttrs: MCC %d, OCMBs attached %d, " + "iv_dimmSize %d GB ", + iv_unitPos, iv_ocmbs.size(), iv_dimmSize); + +fapi_try_exit: + FAPI_DBG("Exiting EffGroupingMccAttrs::getAttrs"); + return fapi2::current_err; +} + + +///---------------------------------------------------------------------------- /// struct EffGroupingMemInfo ///---------------------------------------------------------------------------- /// @@ -704,12 +829,61 @@ fapi_try_exit: /// ---------------------------------------------------------------- /// Total 4 8 /// +/// +/// Axone - 4 MIs total, each MI has 2 MCCs (MC ports/channels). +/// Each channel has two OMI sub-channels that may be +/// connected to an OCMB. +/// Each Explorer ocmb supports up to 2 DIMMs. +/// If both sub-channels have memory connected, the +/// same size must be used for both. +/// Mirroring is done accross sub-channels not +/// accross channels. +/// +/// MI0 --> MCC0 --> OMI0 --> OCMB0 --> DIMM0 +/// --> DIMM1 +/// OMI1 --> OCMB1 --> DIMM0 +/// --> DIMM1 +/// MCC1 --> OMI2 --> OCMB2 --> DIMM0 +/// --> DIMM1 +/// OMI3 --> OCMB3 --> DIMM0 +/// --> DIMM1 +/// +/// MI1 --> MCC2 --> OMI4 --> OCMB4 --> DIMM0 +/// --> DIMM1 +/// OMI5 --> OCMB5 --> DIMM0 +/// --> DIMM1 +/// MCC3 --> OMI6 --> OCMB6 --> DIMM0 +/// --> DIMM1 +/// OMI7 --> OCMB7 --> DIMM0 +/// --> DIMM1 +/// +/// MI2 --> MCC4 --> OMI8 --> OCMB8 --> DIMM0 +/// --> DIMM1 +/// OMI9 --> OCMB9 --> DIMM0 +/// --> DIMM1 +/// MCC5 --> OMI10--> OCMB10--> DIMM0 +/// --> DIMM1 +/// OMI11--> OCMB11--> DIMM0 +/// --> DIMM1 +/// +/// MI3 --> MCC6 --> OMI12--> OCMB12--> DIMM0 +/// --> DIMM1 +/// OMI13--> OCMB13--> DIMM0 +/// --> DIMM1 +/// MCC7 --> OMI14--> OCMB14--> DIMM0 +/// --> DIMM1 +/// OMI15--> OCMB15--> DIMM0 +/// --> DIMM1 +/// ---------------------------------------------------------------- +/// Total 4 8 +/// struct EffGroupingMemInfo { // Constructor EffGroupingMemInfo() { memset(iv_portSize, 0, sizeof(iv_portSize)); + memset(iv_SubChannelsEnabled, 0, sizeof(iv_SubChannelsEnabled)); memset(iv_NVdimmType, false, sizeof(iv_NVdimmType)); } @@ -725,6 +899,8 @@ struct EffGroupingMemInfo // Mark if this proc is a Nimbus bool iv_nimbusProc = false; + // Mark if this proc uses OMI (Axone) + bool iv_omi = false; // Memory sizes behind MC ports uint32_t iv_portSize[NUM_MC_PORTS_PER_PROC]; @@ -735,6 +911,9 @@ struct EffGroupingMemInfo // NVDIMM types behind MC ports bool iv_NVdimmType[NUM_MC_PORTS_PER_PROC]; + // Axone sub-channels enabled per port: 00, 10, 01, 11 + uint8_t iv_SubChannelsEnabled[NUM_MC_PORTS_PER_PROC]; + }; // See doxygen in struct definition. @@ -746,14 +925,29 @@ fapi2::ReturnCode EffGroupingMemInfo::getMemInfo ( // Memory info will be filled in differently for Nimbus vs Cumulus // due to chip structure - // Get the functional MCAs + // Get the functional MCAs (Nimbus) auto l_mcaChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MCA>(); + // Get the functional DMIs (Cumulus) + auto l_dmiChiplets = i_target.getChildren<fapi2::TARGET_TYPE_DMI>(); + + // Get the functional MCCs (Axone) + auto l_mccChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MCC>(); + + + fapi2::ATTR_CHIP_EC_FEATURE_OMI_Type l_omi; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_MAX_INTERLEAVE_GROUP_SIZE, fapi2::Target<fapi2::TARGET_TYPE_SYSTEM>(), iv_maxGroupMemSize), "Error from FAPI_ATTR_GET (ATTR_MAX_INTERLEAVE_GROUP_SIZE)"); + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_OMI, + i_target, + l_omi), + "Error from FAPI_ATTR_GET (ATTR_CHIP_EC_FEATURE_OMI)"); + iv_omi = l_omi; + FAPI_DBG("iv_maxGroupMemSize: 0x%016lX", iv_maxGroupMemSize); if (l_mcaChiplets.size() > 0) @@ -776,36 +970,59 @@ fapi2::ReturnCode EffGroupingMemInfo::getMemInfo ( iv_NVdimmType[l_mcaAttrs.iv_unitPos] = l_mcaAttrs.iv_NVdimmType; } } - else + else if (l_dmiChiplets.size() > 0) { - auto l_dmiChiplets = i_target.getChildren<fapi2::TARGET_TYPE_DMI>(); + FAPI_DBG("Number of DMIs found: %d", l_dmiChiplets.size()); - if (l_dmiChiplets.size() > 0) + // DMI found, proc is a Cumulus. + for (auto l_dmi : l_dmiChiplets) { - FAPI_DBG("Number of DMIs found: %d", l_dmiChiplets.size()); + // Get this DMI attribute info + EffGroupingDmiAttrs l_dmiAttrs; + FAPI_TRY(l_dmiAttrs.getAttrs(l_dmi), + "l_dmiAttrs.getAttrs() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); - // DMI found, proc is a Cumulus. - for (auto l_dmi : l_dmiChiplets) - { - // Get this DMI attribute info - EffGroupingDmiAttrs l_dmiAttrs; - FAPI_TRY(l_dmiAttrs.getAttrs(l_dmi), - "l_dmiAttrs.getAttrs() returns error, l_rc 0x%.8X", - (uint64_t)fapi2::current_err); - - // Fill in memory info - iv_portSize[l_dmiAttrs.iv_unitPos] = l_dmiAttrs.iv_dimmSize; - // No NVDIMM in Cumulus systems (for now) - iv_NVdimmType[l_dmiAttrs.iv_unitPos] = false; - } + // Fill in memory info + iv_portSize[l_dmiAttrs.iv_unitPos] = l_dmiAttrs.iv_dimmSize; + // No NVDIMM in Cumulus systems (for now) + iv_NVdimmType[l_dmiAttrs.iv_unitPos] = false; } - else + } + else if (l_mccChiplets.size() > 0) + { + FAPI_DBG("Number of MCCs found: %d", l_mccChiplets.size()); + + for (auto l_mcc : l_mccChiplets) { - // Note: You may have none of DMI nor MCA but it's a valid state; - // therefore don't flag an error - FAPI_INF("No MCA or DMI found in this proc target"); + // Get this MCC attribute info + EffGroupingMccAttrs l_mccAttrs; + FAPI_TRY(l_mccAttrs.getAttrs(l_mcc), + "l_mccAttrs.getAttrs() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Fill in memory info + iv_portSize[l_mccAttrs.iv_unitPos] = l_mccAttrs.iv_dimmSize; + + for (auto l_omi_pos : l_mccAttrs.iv_omi_pos) + { + iv_SubChannelsEnabled[l_mccAttrs.iv_unitPos] |= + (OMISubChannelConfig::A >> (l_omi_pos % SUBCHANNEL_PER_CHANNEL)); + FAPI_DBG("OMI: l_omi_pos = %d iv_SubChannelsEnabled[%d] = %llx", + l_omi_pos, l_mccAttrs.iv_unitPos, iv_SubChannelsEnabled[l_mccAttrs.iv_unitPos]) + } + + // No NVDIMM in Axone systems (for now - At some point we may have storage class memory) + iv_NVdimmType[l_mccAttrs.iv_unitPos] = false; } } + else + { + // Note: You may have none of DMI, MCC nor MCA but it's a valid state; + // therefore don't flag an error + FAPI_INF("No MCA, DMI, or MCC found in this proc target"); + } + // Display amount of memory for each MC port for (uint8_t ii = 0; ii < NUM_MC_PORTS_PER_PROC; ii++) @@ -837,6 +1054,11 @@ struct EffGroupingData { iv_portGrouped[l_port] = false; } + + for (uint8_t l_grp = 0; l_grp < DATA_GROUPS / 2; l_grp++) + { + iv_OMIMirrorable[l_grp] = false; + } } // The ATTR_MSS_MCS_GROUP_32 attribute @@ -854,6 +1076,12 @@ struct EffGroupingData // Indicates if mirror group is to be created // from data of this non-mirror group uint8_t iv_mirrorOn[DATA_GROUPS / 2]; + + // Indicates if we have OMI type memory + bool iv_omi = false; + // Indicates if all sub-channels in the groups are full + // and therefore mirrorable + bool iv_OMIMirrorable[DATA_GROUPS / 2]; }; @@ -971,16 +1199,26 @@ struct EffGroupingBaseSizeData const EffGroupingSysAttrs& i_sysAttrs, EffGroupingData& io_groupData); + /// + /// @brief getNumMirrorRegions + /// Return the number of mirrored memory regions there are for this chip + /// + /// @return the number of mirrored memory regions + uint64_t getNumMirrorRegions() + { + return iv_omi ? NUM_MIRROR_REGIONS_OMI : NUM_MIRROR_REGIONS; + } + // Public data uint64_t iv_mem_bases[NUM_NON_MIRROR_REGIONS]; uint64_t iv_mem_bases_ack[NUM_NON_MIRROR_REGIONS]; uint64_t iv_memory_sizes[NUM_NON_MIRROR_REGIONS]; uint64_t iv_memory_sizes_ack[NUM_NON_MIRROR_REGIONS]; - uint64_t iv_mirror_bases[NUM_MIRROR_REGIONS]; - uint64_t iv_mirror_bases_ack[NUM_MIRROR_REGIONS]; - uint64_t iv_mirror_sizes[NUM_MIRROR_REGIONS]; - uint64_t iv_mirror_sizes_ack[NUM_MIRROR_REGIONS]; + uint64_t iv_mirror_bases[NUM_MIRROR_REGIONS_MAX]; + uint64_t iv_mirror_bases_ack[NUM_MIRROR_REGIONS_MAX]; + uint64_t iv_mirror_sizes[NUM_MIRROR_REGIONS_MAX]; + uint64_t iv_mirror_sizes_ack[NUM_MIRROR_REGIONS_MAX]; uint64_t iv_smf_bar_base = 0; uint64_t iv_occ_sandbox_base = 0; @@ -989,6 +1227,7 @@ struct EffGroupingBaseSizeData // Num of HTM queues to be reserved for each port uint8_t iv_numHtmQueues[NUM_MC_PORTS_PER_PROC]; + bool iv_omi = false; }; // See description in struct definition @@ -1035,10 +1274,14 @@ void EffGroupingBaseSizeData::setBaseSizeData( // Process mirrored ranges if (i_sysAttrs.iv_hwMirrorEnabled != fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_FALSE) { - for (uint8_t ii = 0; ii < NUM_MIRROR_REGIONS; ii++) + FAPI_DBG("Mirror enabled. Setting values."); + + for (uint8_t ii = 0; ii < getNumMirrorRegions(); ii++) { uint8_t l_index = ii + MIRR_OFFSET; + FAPI_DBG("Mirror enabled. Ports in group %d", i_groupData.iv_data[l_index][PORTS_IN_GROUP]); + if (i_groupData.iv_data[l_index][PORTS_IN_GROUP] != 0) { // Set base address for distinct mirrored ranges @@ -1103,7 +1346,7 @@ uint8_t EffGroupingBaseSizeData::getMemoryRegionIndex( fapi2::ENUM_ATTR_MEM_MIRROR_PLACEMENT_POLICY_FLIPPED) { l_memSizePtr = &iv_mirror_sizes[0] ; - l_numRegions = NUM_MIRROR_REGIONS; + l_numRegions = getNumMirrorRegions(); l_startBaseAddr = iv_mirror_bases[0]; } @@ -1242,7 +1485,7 @@ fapi2::ReturnCode EffGroupingBaseSizeData::set_HTM_OCC_base_addr( } else // Flipped { - l_numRegions = NUM_MIRROR_REGIONS; + l_numRegions = getNumMirrorRegions(); memcpy(l_mem_bases, iv_mirror_bases, sizeof(iv_mirror_bases)); memcpy(l_mem_sizes, iv_mirror_sizes, sizeof(iv_mirror_sizes)); } @@ -1463,7 +1706,7 @@ fapi2::ReturnCode EffGroupingBaseSizeData::setSMFBaseSizeData( } else // Flipped { - l_numRegions = NUM_MIRROR_REGIONS; + l_numRegions = getNumMirrorRegions(); memcpy(l_mem_bases, iv_mirror_bases, sizeof(iv_mirror_bases)); memcpy(l_mem_sizes, iv_mirror_sizes, sizeof(iv_mirror_sizes)); } @@ -1739,26 +1982,26 @@ fapi2::ReturnCode EffGroupingBaseSizeData::setBaseSizeAttr( // Display mirror mode attribute values if (i_sysAttrs.iv_hwMirrorEnabled != fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_FALSE) { - for (uint8_t ii = 0; ii < NUM_MIRROR_REGIONS; ii++) + for (uint8_t ii = 0; ii < getNumMirrorRegions(); ii++) { FAPI_INF("ATTR_PROC_MIRROR_BASES[%u]: 0x%.16llX (%d GB)", ii, iv_mirror_bases[ii], iv_mirror_bases[ii] >> 30); } - for (uint8_t ii = 0; ii < NUM_MIRROR_REGIONS; ii++) + for (uint8_t ii = 0; ii < getNumMirrorRegions(); ii++) { FAPI_INF("ATTR_PROC_MIRROR_BASES_ACK[%u] " "0x%.16llX (%d GB)", ii, iv_mirror_bases_ack[ii], iv_mirror_bases_ack[ii] >> 30); } - for (uint8_t ii = 0; ii < NUM_MIRROR_REGIONS; ii++) + for (uint8_t ii = 0; ii < getNumMirrorRegions(); ii++) { FAPI_INF("ATTR_PROC_MIRROR_SIZES[%u]: 0x%.16llX (%d GB)", ii, iv_mirror_sizes[ii], iv_mirror_sizes[ii] >> 30); } - for (uint8_t ii = 0; ii < NUM_MIRROR_REGIONS; ii++) + for (uint8_t ii = 0; ii < getNumMirrorRegions(); ii++) { FAPI_INF("ATTR_PROC_MIRROR_SIZES_ACK[%u]: 0x%.16llX (%d GB)", ii, iv_mirror_sizes_ack[ii], iv_mirror_sizes_ack[ii] >> 30); @@ -1850,9 +2093,10 @@ fapi_try_exit: /// @param[out] o_groupData Reference to output data /// void grouping_group8PortsPerGroup(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool& i_mirrorRequired) { - // There are 8 MC ports (MCA/DMI) in a proc (Nimbus/Cumulus) and they can + // There are 8 MC ports (MCA/DMI/MCC) in a proc (Nimbus/Cumulus/Axone) and they can // be grouped together if they all have the same memory size per ports // and there is no mix of NVDIMM/RDIMM between ports. FAPI_DBG("Entering"); @@ -1868,6 +2112,8 @@ void grouping_group8PortsPerGroup(const EffGroupingMemInfo& i_memInfo, for (uint8_t l_pos = 1; l_pos < NUM_MC_PORTS_PER_PROC; l_pos++) { if ( (i_memInfo.iv_portSize[0] != i_memInfo.iv_portSize[l_pos]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[0] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[l_pos] == OMISubChannelConfig::BOTH))) || (i_memInfo.iv_NVdimmType[0] != i_memInfo.iv_NVdimmType[l_pos]) ) { // This port does not have the same memory size as port 0, or @@ -1876,6 +2122,8 @@ void grouping_group8PortsPerGroup(const EffGroupingMemInfo& i_memInfo, FAPI_DBG("Can not group by 8: "); FAPI_DBG(" i_memInfo.iv_portSize[0] = %d GB, i_memInfo.iv_portSize[%d] = %d GB", i_memInfo.iv_portSize[0], l_pos, i_memInfo.iv_portSize[l_pos]); + FAPI_DBG(" mirrorReq = %d i_memInfo.iv_SubChannelsEnabled[0] = %d, i_memInfo.iv_SubChannelsEnabled[%d] = %d", + i_mirrorRequired, i_memInfo.iv_SubChannelsEnabled[0], l_pos, i_memInfo.iv_SubChannelsEnabled[l_pos]); FAPI_DBG(" i_memInfo.iv_NVdimmType[0] = %d, i_memInfo.iv_NVdimmType[%d] = %d", i_memInfo.iv_NVdimmType[0], l_pos, i_memInfo.iv_NVdimmType[l_pos]); grouped = false; @@ -1902,9 +2150,17 @@ void grouping_group8PortsPerGroup(const EffGroupingMemInfo& i_memInfo, g++; // increase o_groupData.iv_numGroups // Record which MC ports were grouped + // Check if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + for (uint8_t ii = 0; ii < NUM_MC_PORTS_PER_PROC; ii++) { o_groupData.iv_portGrouped[ii] = true; + + if (o_groupData.iv_OMIMirrorable[g] && i_memInfo.iv_SubChannelsEnabled[ii] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } } FAPI_INF("grouping_group8PortsPerGroup: Successfully grouped 8 " @@ -1931,7 +2187,8 @@ void grouping_group8PortsPerGroup(const EffGroupingMemInfo& i_memInfo, /// @param[out] o_groupData Reference to output data /// void grouping_group6PortsPerGroup(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool& i_mirrorRequired) { FAPI_DBG("Entering"); @@ -1979,6 +2236,8 @@ void grouping_group6PortsPerGroup(const EffGroupingMemInfo& i_memInfo, if ( (o_groupData.iv_portGrouped[CFG_6MCPORT[ii][jj]]) || (i_memInfo.iv_portSize[CFG_6MCPORT[ii][0]] != i_memInfo.iv_portSize[CFG_6MCPORT[ii][jj]]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[ii][0]] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[ii][jj]] == OMISubChannelConfig::BOTH))) || (i_memInfo.iv_NVdimmType[CFG_6MCPORT[ii][0]] != i_memInfo.iv_NVdimmType[CFG_6MCPORT[ii][jj]]) ) { @@ -1993,6 +2252,11 @@ void grouping_group6PortsPerGroup(const EffGroupingMemInfo& i_memInfo, ii, i_memInfo.iv_portSize[CFG_6MCPORT[ii][0]]); FAPI_DBG(" i_memInfo.iv_portSize[CFG_6MCPORT[%d][%d]] = %d GB", ii, jj, i_memInfo.iv_portSize[CFG_6MCPORT[ii][jj]]); + // Display subchannels of first port vs port jj in row + FAPI_DBG(" i_mirrorReq = %d i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[%d][0]] = %d", + i_mirrorRequired, ii, i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[ii][0]]); + FAPI_DBG(" i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[%d][%d]] = %d", + ii, jj, i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[ii][jj]]); // Display DIMM type of first port vs port jj in row FAPI_DBG(" i_memInfo.iv_NVdimmType[CFG_6MCPORT[%d][0]] = %d", ii, i_memInfo.iv_NVdimmType[CFG_6MCPORT[ii][0]]); @@ -2015,10 +2279,19 @@ void grouping_group6PortsPerGroup(const EffGroupingMemInfo& i_memInfo, PORTS_PER_GROUP * i_memInfo.iv_portSize[CFG_6MCPORT[ii][0]]; // Record which MC ports were grouped + // Check if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + for (uint8_t jj = 0; jj < PORTS_PER_GROUP; jj++) { o_groupData.iv_data[g][MEMBER_IDX(jj)] = CFG_6MCPORT[ii][jj]; o_groupData.iv_portGrouped[CFG_6MCPORT[ii][jj]] = true; + + if (o_groupData.iv_OMIMirrorable[g] + && i_memInfo.iv_SubChannelsEnabled[CFG_6MCPORT[ii][jj]] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } } g++; @@ -2051,7 +2324,8 @@ void grouping_group6PortsPerGroup(const EffGroupingMemInfo& i_memInfo, /// @param[out] o_groupData Reference to output data /// void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool& i_mirrorRequired) { FAPI_DBG("Entering"); @@ -2108,6 +2382,8 @@ void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, if ( (o_groupData.iv_portGrouped[CFG_4MCPORT[ii][jj]]) || (i_memInfo.iv_portSize[CFG_4MCPORT[ii][0]] != i_memInfo.iv_portSize[CFG_4MCPORT[ii][jj]]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[ii][0]] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[ii][jj]] == OMISubChannelConfig::BOTH))) || (i_memInfo.iv_NVdimmType[CFG_4MCPORT[ii][0]] != i_memInfo.iv_NVdimmType[CFG_4MCPORT[ii][jj]]) ) { @@ -2122,6 +2398,11 @@ void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, ii, i_memInfo.iv_portSize[CFG_4MCPORT[ii][0]]); FAPI_DBG(" i_memInfo.iv_portSize[CFG_4MCPORT[%d][%d]] = %d GB", ii, jj, i_memInfo.iv_portSize[CFG_4MCPORT[ii][jj]]); + // Display subchannels of first port vs port jj in row + FAPI_DBG(" i_mirrorReq = %d i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[%d][0]] = %d", + i_mirrorRequired, ii, i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[ii][0]]); + FAPI_DBG(" i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[%d][%d]] = %d", + ii, jj, i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[ii][jj]]); // Display DIMM type of first port vs port jj in row FAPI_DBG(" i_memInfo.iv_NVdimmType[CFG_4MCPORT[%d][0]] = %d", ii, i_memInfo.iv_NVdimmType[CFG_4MCPORT[ii][0]]); @@ -2184,14 +2465,24 @@ void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, o_groupData.iv_data[g][MEMBER_IDX(1)] = CFG_4MCPORT[gp1][2]; o_groupData.iv_data[g][MEMBER_IDX(2)] = CFG_4MCPORT[gp1][1]; o_groupData.iv_data[g][MEMBER_IDX(3)] = CFG_4MCPORT[gp1][3]; - g++; // Record which MC ports were grouped + // Check if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + for (uint8_t ii = 0; ii < PORTS_PER_GROUP; ii++) { o_groupData.iv_portGrouped[CFG_4MCPORT[gp1][ii]] = true; + + if (o_groupData.iv_OMIMirrorable[g] && + i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[gp1][ii]] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } } + g++; + FAPI_INF("grouping_group4PortsPerGroup: Successfully grouped 4 " "MC ports. CFG_4MCPORT[%d] %u, %u, %u, %u", gp1, CFG_4MCPORT[gp1][0], CFG_4MCPORT[gp1][1], @@ -2210,14 +2501,24 @@ void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, o_groupData.iv_data[g][MEMBER_IDX(1)] = CFG_4MCPORT[gp2][2]; o_groupData.iv_data[g][MEMBER_IDX(2)] = CFG_4MCPORT[gp2][1]; o_groupData.iv_data[g][MEMBER_IDX(3)] = CFG_4MCPORT[gp2][3]; - g++; // Record which MC ports were grouped + // Check if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + for (uint8_t ii = 0; ii < PORTS_PER_GROUP; ii++) { o_groupData.iv_portGrouped[CFG_4MCPORT[gp2][ii]] = true; + + if (o_groupData.iv_OMIMirrorable[g] && + i_memInfo.iv_SubChannelsEnabled[CFG_4MCPORT[gp2][ii]] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } } + g++; + FAPI_INF("grouping_group4PortsPerGroup: Successfully grouped 4 " "MC ports. CFG_4MCPORT[%d] %u, %u, %u, %u", gp2, CFG_4MCPORT[gp2][0], CFG_4MCPORT[gp2][1], @@ -2239,11 +2540,13 @@ void grouping_group4PortsPerGroup(const EffGroupingMemInfo& i_memInfo, /// - iv_portGrouped[<group>] /// - iv_numGroups /// -/// @param[in] i_memInfo Reference to EffGroupingMemInfo structure -/// @param[out] o_groupData Reference to output data +/// @param[in] i_memInfo Reference to EffGroupingMemInfo structure +/// @param[out] o_groupData Reference to output data +/// @param[in] i_mirrorRequired Mirroring is required by the system /// void grouping_group3PortsPerGroup(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool& i_mirrorRequired) { FAPI_DBG("Entering"); @@ -2325,8 +2628,10 @@ void grouping_group3PortsPerGroup(const EffGroupingMemInfo& i_memInfo, // Skip if this port is already grouped or has different // amount of memory or dimm type if ( (o_groupData.iv_portGrouped[CFG_3MCPORT[ii][jj]]) || - (i_memInfo.iv_portSize[CFG_3MCPORT[ii][0]] != - i_memInfo.iv_portSize[CFG_3MCPORT[ii][jj]]) || + (i_memInfo.iv_portSize[CFG_3MCPORT[ii][0]] != i_memInfo.iv_portSize[CFG_3MCPORT[ii][jj]]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[ii][0]] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[ii][jj]] == OMISubChannelConfig::BOTH)) + ) || (i_memInfo.iv_NVdimmType[CFG_3MCPORT[ii][0]] != i_memInfo.iv_NVdimmType[CFG_3MCPORT[ii][jj]]) ) { @@ -2365,6 +2670,10 @@ void grouping_group3PortsPerGroup(const EffGroupingMemInfo& i_memInfo, ii, i_memInfo.iv_portSize[CFG_3MCPORT[ii][0]]); FAPI_DBG(" i_memInfo.iv_portSize[CFG_3MCPORT[%d][%d]] = %d GB", ii, jj, i_memInfo.iv_portSize[CFG_3MCPORT[ii][jj]]); + FAPI_DBG(" i_mirrorReq = %d i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[%d][0]] = %d", + i_mirrorRequired, ii, i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[ii][0]]); + FAPI_DBG(" i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[%d][%d]] = %d", + ii, jj, i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[ii][jj]]); FAPI_DBG(" i_memInfo.iv_NVdimmType[CFG_3MCPORT[%d][0]] = %d", ii, i_memInfo.iv_NVdimmType[CFG_3MCPORT[ii][0]]); FAPI_DBG(" i_memInfo.iv_NVdimmType[CFG_3MCPORT[%d][%d]] = %d", @@ -2380,14 +2689,24 @@ void grouping_group3PortsPerGroup(const EffGroupingMemInfo& i_memInfo, o_groupData.iv_data[g][MEMBER_IDX(0)] = CFG_3MCPORT[ii][0]; o_groupData.iv_data[g][MEMBER_IDX(1)] = CFG_3MCPORT[ii][1]; o_groupData.iv_data[g][MEMBER_IDX(2)] = CFG_3MCPORT[ii][2]; - g++; - // Record which MC ports were grouped + // Record which MC ports were grouped, + // Check if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + for (uint8_t jj = 0; jj < PORTS_PER_GROUP; jj++) { o_groupData.iv_portGrouped[CFG_3MCPORT[ii][jj]] = true; + + if (o_groupData.iv_OMIMirrorable[g] && + i_memInfo.iv_SubChannelsEnabled[CFG_3MCPORT[ii][jj]] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } } + g++; + FAPI_INF("grouping_group3PortsPerGroup: Successfully grouped 3 " "MC ports. CFG_3MCPORT[%d] %u, %u, %u, %u", ii, CFG_3MCPORT[ii][0], CFG_3MCPORT[ii][1], @@ -2418,7 +2737,8 @@ void grouping_group3PortsPerGroup(const EffGroupingMemInfo& i_memInfo, /// @param[out] o_groupData Reference to output data /// void grouping_2ports_same_MCS(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool i_mirrorRequired) { FAPI_DBG("Entering"); FAPI_INF("grouping_2ports_same_MCS: Attempting to group 2 ports on same MCS"); @@ -2440,6 +2760,8 @@ void grouping_2ports_same_MCS(const EffGroupingMemInfo& i_memInfo, // Check 2nd port if ( (o_groupData.iv_portGrouped[pos + 1]) || (i_memInfo.iv_portSize[pos + 1] != i_memInfo.iv_portSize[pos]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[pos + 1] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[pos] == OMISubChannelConfig::BOTH))) || (i_memInfo.iv_NVdimmType[pos + 1] != i_memInfo.iv_NVdimmType[pos]) ) { FAPI_DBG("Port %u already grouped or has different memory size/type, skip", @@ -2449,6 +2771,9 @@ void grouping_2ports_same_MCS(const EffGroupingMemInfo& i_memInfo, FAPI_DBG(" i_memInfo.iv_portSize[%d] = %d GB; i_memInfo.iv_portSize[%d] = %d GB", pos, i_memInfo.iv_portSize[pos], pos + 1, i_memInfo.iv_portSize[pos + 1]); + FAPI_DBG(" i_mirrorReq = %d i_memInfo.iv_SubChannelsEnabled[%d] = %d; i_memInfo.iv_SubChannelsEnabled[%d] = %d", + i_mirrorRequired, pos, i_memInfo.iv_portSize[pos], + pos + 1, i_memInfo.iv_portSize[pos + 1]); FAPI_DBG(" i_memInfo.iv_NVdimmType[%d] = %d; i_memInfo.iv_NVdimmType[%d] = %d", pos, i_memInfo.iv_NVdimmType[pos], pos + 1, i_memInfo.iv_NVdimmType[pos + 1]); @@ -2470,12 +2795,21 @@ void grouping_2ports_same_MCS(const EffGroupingMemInfo& i_memInfo, PORTS_PER_GROUP * i_memInfo.iv_portSize[pos]; o_groupData.iv_data[g][MEMBER_IDX(0)] = pos; o_groupData.iv_data[g][MEMBER_IDX(1)] = pos + 1; - g++; // Record which MC ports were grouped o_groupData.iv_portGrouped[pos] = true; o_groupData.iv_portGrouped[pos + 1] = true; + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + + if (i_memInfo.iv_SubChannelsEnabled[pos] != OMISubChannelConfig::BOTH || + i_memInfo.iv_SubChannelsEnabled[pos + 1] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } + + g++; + FAPI_INF("grouping_2ports_same_MCS: Successfully grouped " "MC ports: %u, %u", pos, pos + 1); } @@ -2503,7 +2837,8 @@ void grouping_2ports_same_MCS(const EffGroupingMemInfo& i_memInfo, /// @param[out] o_groupData Reference to output data /// void grouping_2groupsOf2_cross_MCS(const EffGroupingMemInfo& i_memInfo, - EffGroupingData& o_groupData) + EffGroupingData& o_groupData, + const bool& i_mirrorRequired) { FAPI_DBG("Entering"); FAPI_INF("grouping_2groupsOf2_cross_MCS: Attempting to group 2 groups of 2 on cross-MCS"); @@ -2570,6 +2905,8 @@ void grouping_2groupsOf2_cross_MCS(const EffGroupingMemInfo& i_memInfo, if ( (i_memInfo.iv_portSize[mcs1pos0] == i_memInfo.iv_portSize[mcs2pos0]) && (i_memInfo.iv_NVdimmType[mcs1pos0] == i_memInfo.iv_NVdimmType[mcs2pos0]) && + ((!i_mirrorRequired) || ((i_memInfo.iv_SubChannelsEnabled[mcs1pos0] == OMISubChannelConfig::BOTH) == + (i_memInfo.iv_SubChannelsEnabled[mcs2pos0] == OMISubChannelConfig::BOTH))) && (i_memInfo.iv_portSize[mcs1pos0 + 1] == i_memInfo.iv_portSize[mcs2pos0 + 1]) && (i_memInfo.iv_NVdimmType[mcs1pos0 + 1] == i_memInfo.iv_NVdimmType[mcs2pos0 + 1]) ) { @@ -2581,6 +2918,8 @@ void grouping_2groupsOf2_cross_MCS(const EffGroupingMemInfo& i_memInfo, } else if ( (i_memInfo.iv_portSize[mcs1pos0] == i_memInfo.iv_portSize[mcs2pos0 + 1]) && (i_memInfo.iv_NVdimmType[mcs1pos0] == i_memInfo.iv_NVdimmType[mcs2pos0 + 1]) && + ((!i_mirrorRequired) || ((i_memInfo.iv_SubChannelsEnabled[mcs1pos0] == OMISubChannelConfig::BOTH) == + (i_memInfo.iv_SubChannelsEnabled[mcs2pos0 + 1] == OMISubChannelConfig::BOTH))) && (i_memInfo.iv_portSize[mcs1pos0 + 1] == i_memInfo.iv_portSize[mcs2pos0]) && (i_memInfo.iv_NVdimmType[mcs1pos0 + 1] == i_memInfo.iv_NVdimmType[mcs2pos0]) ) { @@ -2624,11 +2963,23 @@ void grouping_2groupsOf2_cross_MCS(const EffGroupingMemInfo& i_memInfo, o_groupData.iv_portGrouped[l_twoGroupOf2[1][0]] = true; o_groupData.iv_portGrouped[l_twoGroupOf2[1][1]] = true; + // See if OMI mirrorable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + + if (i_memInfo.iv_SubChannelsEnabled[l_twoGroupOf2[0][0]] != OMISubChannelConfig::BOTH || + i_memInfo.iv_SubChannelsEnabled[l_twoGroupOf2[0][1]] != OMISubChannelConfig::BOTH || + i_memInfo.iv_SubChannelsEnabled[l_twoGroupOf2[1][0]] != OMISubChannelConfig::BOTH || + i_memInfo.iv_SubChannelsEnabled[l_twoGroupOf2[1][1]] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } + FAPI_INF("grouping_2groupsOf2_cross_MCS: Successfully grouped " "2 groups of 2 from MCS %u and %u", mcs1, mcs2); FAPI_INF(" Group: Ports %u and %u; Group: ports %u and %u", l_twoGroupOf2[0][0], l_twoGroupOf2[0][1], l_twoGroupOf2[1][0], l_twoGroupOf2[1][1]); + // Break out of mcs2 loop break; } @@ -2651,12 +3002,12 @@ void grouping_2groupsOf2_cross_MCS(const EffGroupingMemInfo& i_memInfo, /// /// @param[in] i_memInfo Reference to EffGroupingMemInfo structure /// @param[out] o_groupData Reference to output data -/// @param[in] i_disallow_cross_mcs Disallow grouping of ports which are not +/// @param[in] i_mirrorRequired Mirroring required, and disallow grouping of ports which are not /// in the same MCS /// void grouping_group2PortsPerGroup(const EffGroupingMemInfo& i_memInfo, EffGroupingData& o_groupData, - const bool& i_disallow_cross_mcs) + const bool& i_mirrorRequired) { FAPI_DBG("Entering"); FAPI_INF("grouping_group2PortsPerGroup: Attempting to group 2 MC ports"); @@ -2665,16 +3016,16 @@ void grouping_group2PortsPerGroup(const EffGroupingMemInfo& i_memInfo, uint8_t l_otherPort = 0; // 1. Try to group 2 ports that are in the same MCS (highest priority) - grouping_2ports_same_MCS(i_memInfo, o_groupData); + grouping_2ports_same_MCS(i_memInfo, o_groupData, i_mirrorRequired); // do not permit cross-MCS grouping - if (i_disallow_cross_mcs) + if (i_mirrorRequired) { goto fapi_try_exit; } // 2. Try two groups of 2 on cross-MCS - grouping_2groupsOf2_cross_MCS(i_memInfo, o_groupData); + grouping_2groupsOf2_cross_MCS(i_memInfo, o_groupData, i_mirrorRequired); // 3. Attempt group of 2 for the remaining un-grouped ports (cross-MCS) FAPI_INF("Attempting to group the remaining ports as group of 2"); @@ -2734,7 +3085,9 @@ void grouping_group2PortsPerGroup(const EffGroupingMemInfo& i_memInfo, // Can not group if this port already grouped or has different memory size if ( (o_groupData.iv_portGrouped[ii]) || - (i_memInfo.iv_portSize[ii] != i_memInfo.iv_portSize[pos]) ) + (i_memInfo.iv_portSize[ii] != i_memInfo.iv_portSize[pos]) || + (i_mirrorRequired && ((i_memInfo.iv_SubChannelsEnabled[ii] == OMISubChannelConfig::BOTH) != + (i_memInfo.iv_SubChannelsEnabled[pos] == OMISubChannelConfig::BOTH)) )) { FAPI_DBG("Skip port %u, it's already grouped or memsize is not equal", ii); continue; @@ -2764,12 +3117,21 @@ void grouping_group2PortsPerGroup(const EffGroupingMemInfo& i_memInfo, PORTS_PER_GROUP * i_memInfo.iv_portSize[pos]; o_groupData.iv_data[g][MEMBER_IDX(0)] = pos; o_groupData.iv_data[g][MEMBER_IDX(1)] = ii; - g++; + // See if OMI Mirrable + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi; + + if (i_memInfo.iv_SubChannelsEnabled[pos] != OMISubChannelConfig::BOTH || + i_memInfo.iv_SubChannelsEnabled[ii] != OMISubChannelConfig::BOTH) + { + o_groupData.iv_OMIMirrorable[g] = false; + } // Record which MC ports were grouped o_groupData.iv_portGrouped[pos] = true; o_groupData.iv_portGrouped[ii] = true; + g++; + FAPI_INF("grouping_group2PortsPerGroup: Successfully grouped 2 " "MC ports: %u, %u", pos, ii); @@ -2816,6 +3178,9 @@ void grouping_group1PortsPerGroup(const EffGroupingMemInfo& i_memInfo, o_groupData.iv_data[g][PORTS_IN_GROUP] = 1; o_groupData.iv_data[g][GROUP_SIZE] = i_memInfo.iv_portSize[pos]; o_groupData.iv_data[g][MEMBER_IDX(0)] = pos; + o_groupData.iv_OMIMirrorable[g] = o_groupData.iv_omi && + (i_memInfo.iv_SubChannelsEnabled[pos] == OMISubChannelConfig::BOTH); + g++; // Record which MCS was grouped @@ -2933,6 +3298,57 @@ void getAttachedDimms( FAPI_DBG("End"); } +/// template specialization for OCMB target type +template <> +void getAttachedDimms( + const fapi2::Target<fapi2::TARGET_TYPE_OCMB_CHIP>& i_target, + std::vector<fapi2::Target<fapi2::TARGET_TYPE_DIMM>>& o_dimm_targets) +{ + FAPI_DBG("Start"); + std::vector<fapi2::Target<fapi2::TARGET_TYPE_DIMM>> l_dimm_targets = + i_target.template getChildren<fapi2::TARGET_TYPE_DIMM>(); + + o_dimm_targets.insert(o_dimm_targets.end(), + l_dimm_targets.begin(), + l_dimm_targets.end()); + FAPI_DBG("End"); +} + +/// template specialization for DMI target type +template <> +void getAttachedDimms( + const fapi2::Target<fapi2::TARGET_TYPE_OMI>& i_target, + std::vector<fapi2::Target<fapi2::TARGET_TYPE_DIMM>>& o_dimm_targets) +{ + FAPI_DBG("Start"); + + // determine attached Centaur chip + for (auto l_ocmb_target : i_target.template getChildren<fapi2::TARGET_TYPE_OCMB_CHIP>()) + { + // get set of valid MBAs + getAttachedDimms<fapi2::TARGET_TYPE_OCMB_CHIP>(l_ocmb_target, o_dimm_targets); + } + + FAPI_DBG("End"); +} + +/// template specialization for DMI target type +template <> +void getAttachedDimms( + const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_target, + std::vector<fapi2::Target<fapi2::TARGET_TYPE_DIMM>>& o_dimm_targets) +{ + FAPI_DBG("Start"); + + // determine attached Centaur chip + for (auto l_omi_target : i_target.template getChildren<fapi2::TARGET_TYPE_OMI>()) + { + getAttachedDimms<fapi2::TARGET_TYPE_OMI>(l_omi_target, o_dimm_targets); + } + + FAPI_DBG("End"); +} + /// template specialization for DMI target type template <> void getAttachedDimms( @@ -3345,6 +3761,61 @@ fapi_try_exit: } + +/// template specialization for MCC target type +template<> +fapi2::ReturnCode calloutDimmsForUngroupedPorts( + const uint8_t i_ports_functional[NUM_MC_PORTS_PER_PROC], + const uint8_t i_ports_ungrouped[NUM_MC_PORTS_PER_PROC], + const uint8_t i_ports_tgt_index[NUM_MC_PORTS_PER_PROC], + const std::vector<fapi2::Target<fapi2::TARGET_TYPE_MCC>>& i_port_targets, + const EffGroupingMemInfo& i_memInfo, + const uint8_t i_hwMirrorEnabled, + std::vector<fapi2::Target<fapi2::TARGET_TYPE_DIMM>>& o_dimm_targets, + fapi2::ReturnCode& o_rc) +{ + FAPI_DBG("Start"); + + // o_rc contains the error we're going to append to + fapi2::current_err = fapi2::FAPI2_RC_SUCCESS; + + // callout each DIMM which is associated with each ungrouped port + for (uint8_t ii = 0; (ii < NUM_MC_PORTS_PER_PROC); ii += 1) + { + if (i_ports_functional[ii] && i_ports_ungrouped[ii]) + { + getAttachedDimms(i_port_targets[i_ports_tgt_index[ii]], + o_dimm_targets); + } + } + + // add DIMM callouts + for (const auto& l_dimm_target : o_dimm_targets) + { + fapi2::Target<fapi2::TARGET_TYPE_MCC> l_port_target = l_dimm_target + .getParent<fapi2::TARGET_TYPE_OCMB_CHIP>() + .getParent<fapi2::TARGET_TYPE_OMI>() + .getParent<fapi2::TARGET_TYPE_MCC>(); + uint8_t l_port_index = 0; + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, + l_port_target, + l_port_index), + "Error from FAPI_ATTR_GET (ATTR_CHIP_UNIT_POS)"); + + calloutDIMM(l_dimm_target, + l_port_target, + l_port_index, + i_memInfo.iv_portSize[l_port_index], + o_rc); + } + +fapi_try_exit: + FAPI_DBG("End"); + return fapi2::current_err; +} + + /// /// @brief Finds ungrouped ports /// @@ -3629,24 +4100,37 @@ void setupMirrorGroup( // Loop thru groups to see if mirror group is possible for (uint8_t l_group = 0; l_group < io_groupData.iv_numGroups; l_group++) { - // If group of 4, 6, or 8, mirror is allowed - // Note: For group of 4/6/8, the ports are always in the same MC - // port pair per design. - if ( (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 4) || - (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 6) || - (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 8) ) + if (io_groupData.iv_omi) { - io_groupData.iv_mirrorOn[l_group] = 1; - } + FAPI_DBG("io_groupData.iv_omi: true io_groupData.iv_OMIMirrorable[%d] = %d", + l_group, io_groupData.iv_OMIMirrorable[l_group]); - // For group of 2, determine if both ports are in the same MCS/MI - else if (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 2) + if (io_groupData.iv_OMIMirrorable[l_group]) + { + io_groupData.iv_mirrorOn[l_group] = 1; + } + } + else { - if ( (io_groupData.iv_data[l_group][MEMBER_IDX(0)] / 2) == - (io_groupData.iv_data[l_group][MEMBER_IDX(1)] / 2) ) + // If group of 4, 6, or 8, mirror is allowed + // Note: For group of 4/6/8, the ports are always in the same MC + // port pair per design. + if ( (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 4) || + (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 6) || + (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 8)) { io_groupData.iv_mirrorOn[l_group] = 1; } + + // For group of 2, determine if both ports are in the same MCS/MI + else if (io_groupData.iv_data[l_group][PORTS_IN_GROUP] == 2) + { + if ( (io_groupData.iv_data[l_group][MEMBER_IDX(0)] / 2) == + (io_groupData.iv_data[l_group][MEMBER_IDX(1)] / 2) ) + { + io_groupData.iv_mirrorOn[l_group] = 1; + } + } } FAPI_INF("setupMirrorGroup: Group %d, PortsInGroup %d, Mirror = %d", @@ -3706,6 +4190,7 @@ fapi2::ReturnCode grouping_calcRegions( if (i_cfgMirror) { l_cur_m_base_addr = i_procAttrs.iv_mirrorBaseAddr[l_cur_m_region_idx]; + FAPI_DBG("l_cur_m_base_addr %016llx", l_cur_m_base_addr); for (uint8_t pos = 0; pos < io_groupData.iv_numGroups; pos++) { @@ -3747,7 +4232,8 @@ fapi2::ReturnCode grouping_calcRegions( for (uint8_t pos = 0; pos < io_groupData.iv_numGroups; pos++) { bool l_map_mirror = i_cfgMirror && - (io_groupData.iv_data[pos][PORTS_IN_GROUP] > 1); + ((io_groupData.iv_data[pos][PORTS_IN_GROUP] > 1) || + io_groupData.iv_OMIMirrorable[pos]); FAPI_DBG("pos: %d, l_map_mirror: %d", pos, l_map_mirror); @@ -3912,9 +4398,16 @@ fapi2::ReturnCode grouping_calcRegions( if (l_map_mirror) { + FAPI_DBG("io_groupData.iv_data[pos + MIRR_OFFSET][BASE_ADDR]: %016llx", + io_groupData.iv_data[pos + MIRR_OFFSET][BASE_ADDR]); + FAPI_DBG("io_groupData.iv_data[pos + MIRR_OFFSET][GROUP_SIZE]: %016llx", + io_groupData.iv_data[pos + MIRR_OFFSET][GROUP_SIZE]); + FAPI_DBG("io_groupData.iv_data[pos + MIRR_OFFSET][ALT_SIZE(ii)]: %016llx", + io_groupData.iv_data[pos + MIRR_OFFSET][ALT_SIZE(ii)]); io_groupData.iv_data[pos + MIRR_OFFSET][ALT_BASE_ADDR(ii)] = io_groupData.iv_data[pos + MIRR_OFFSET][BASE_ADDR] + - io_groupData.iv_data[pos + MIRR_OFFSET][GROUP_SIZE] / 2; + io_groupData.iv_data[pos + MIRR_OFFSET][GROUP_SIZE] - + io_groupData.iv_data[pos + MIRR_OFFSET][ALT_SIZE(ii)]; io_groupData.iv_data[pos + MIRR_OFFSET][ALT_VALID(ii)] = 1; } } @@ -4066,6 +4559,7 @@ fapi2::ReturnCode p9_mss_eff_grouping( EffGroupingBaseSizeData l_baseSizeData; EffGroupingData l_groupData; bool l_mirrorIsOn = false; + bool l_mirrorReq = false; // ---------------------------------------------- // Get the attributes needed for memory grouping @@ -4095,6 +4589,13 @@ fapi2::ReturnCode p9_mss_eff_grouping( "p9_mss_eff_grouping: l_memInfo.get_memInfo() returns an error, " "l_rc 0x%.8X", (uint64_t)fapi2::current_err); + // ------------------------------------------------------------------ + // Update groupings allowed - set omi + // ------------------------------------------------------------------ + l_sysAttrs.updateGroupsAllowed(l_memInfo.iv_omi); + l_baseSizeData.iv_omi = l_memInfo.iv_omi; + l_groupData.iv_omi = l_memInfo.iv_omi; + // ---------------------------------------------------------------------- // Attempt to group the memory per Group port (per MCA/DMI). // P9 MC architecture allows 1, 2, 3, 4, 6, or 8 MC ports to be grouped @@ -4105,25 +4606,27 @@ fapi2::ReturnCode p9_mss_eff_grouping( // ---------------------------------------------------------------------- FAPI_INF("Attempt memory grouping"); + l_mirrorReq = (l_sysAttrs.iv_hwMirrorEnabled == fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_TRUE); + // Group MCs if (l_sysAttrs.iv_groupsAllowed & GROUP_8) { - grouping_group8PortsPerGroup(l_memInfo, l_groupData); + grouping_group8PortsPerGroup(l_memInfo, l_groupData, l_mirrorReq); } if (l_sysAttrs.iv_groupsAllowed & GROUP_6) { - grouping_group6PortsPerGroup(l_memInfo, l_groupData); + grouping_group6PortsPerGroup(l_memInfo, l_groupData, l_mirrorReq); } if (l_sysAttrs.iv_groupsAllowed & GROUP_4) { - grouping_group4PortsPerGroup(l_memInfo, l_groupData); + grouping_group4PortsPerGroup(l_memInfo, l_groupData, l_mirrorReq); } if (l_sysAttrs.iv_groupsAllowed & GROUP_3) { - grouping_group3PortsPerGroup(l_memInfo, l_groupData); + grouping_group3PortsPerGroup(l_memInfo, l_groupData, l_mirrorReq); } if (l_sysAttrs.iv_groupsAllowed & GROUP_2) @@ -4131,7 +4634,7 @@ fapi2::ReturnCode p9_mss_eff_grouping( grouping_group2PortsPerGroup( l_memInfo, l_groupData, - (l_sysAttrs.iv_hwMirrorEnabled == fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_TRUE)); + l_mirrorReq); } if (l_sysAttrs.iv_groupsAllowed & GROUP_1) @@ -4149,6 +4652,15 @@ fapi2::ReturnCode p9_mss_eff_grouping( "grouping_findUngroupedPorts() returns an error, l_rc 0x%.8X", (uint64_t)fapi2::current_err); } + else if (l_memInfo.iv_omi) + { + FAPI_TRY(grouping_findUngroupedPorts<fapi2::TARGET_TYPE_MCC>(i_target, + l_memInfo, + l_groupData, + l_sysAttrs.iv_hwMirrorEnabled), + "grouping_findUngroupedPorts() returns an error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + } else { FAPI_TRY(grouping_findUngroupedPorts<fapi2::TARGET_TYPE_DMI>(i_target, diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.H b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.H index 1bb1d0f71..ca0ec14c6 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.H +++ b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.H @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2015,2018 */ +/* Contributors Listed Below - COPYRIGHT 2015,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -56,6 +56,7 @@ const uint8_t DATA_GROUPS = 16; // 8 regular groups, 8 mirrored groups const uint8_t MIRR_OFFSET = 8; // Start of mirrored offset in DATA_GROUPS const uint8_t DATA_ELEMENTS = 21; // 21 items of data for each group +const uint8_t SUBCHANNEL_PER_CHANNEL = 2; // Number of OMI's per MCC // Indexes used for EffGroupingData::iv_data DATA ELEMENTS const uint8_t PORT_SIZE = 0; // Memory size of each port in group (GB) @@ -73,6 +74,8 @@ const uint8_t SMF_BASE_ADDR = 20; // SMF Base Address // Number of memory regions const uint8_t NUM_NON_MIRROR_REGIONS = 8; const uint8_t NUM_MIRROR_REGIONS = 4; +const uint8_t NUM_MIRROR_REGIONS_OMI = 8; // Each port does its own mirroring +const uint8_t NUM_MIRROR_REGIONS_MAX = NUM_MIRROR_REGIONS_OMI; // Max mirror regions const uint8_t NUM_OF_CHTM_REGIONS = 24; // 24 CHTM memory regions const uint8_t NUM_OF_ALT_MEM_REGIONS = 2; // 2 memory holes diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.mk b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.mk index d960a266a..bde482fa3 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.mk +++ b/src/import/chips/p9/procedures/hwp/nest/p9_mss_eff_grouping.mk @@ -5,7 +5,7 @@ # # OpenPOWER HostBoot Project # -# Contributors Listed Below - COPYRIGHT 2015,2018 +# Contributors Listed Below - COPYRIGHT 2015,2019 # [+] International Business Machines Corp. # # @@ -27,6 +27,7 @@ PROCEDURE=p9_mss_eff_grouping $(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9/procedures/hwp/memory/) lib$(PROCEDURE)_DEPLIBS+=cen +lib$(PROCEDURE)_DEPLIBS+=mss_p9a OBJS+=p9_fbc_utils.o $(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)) $(call BUILD_PROCEDURE) diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.C b/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.C index 1138a3447..cb80ca77b 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.C +++ b/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.C @@ -44,9 +44,13 @@ #include <p9_mc_scom_addresses_fld.H> #include <p9n2_mc_scom_addresses.H> #include <p9n2_mc_scom_addresses_fld.H> +#include <p9a_misc_scom_addresses.H> +#include <p9a_misc_scom_addresses_fld.H> +#include <p9a_addr_ext.H> #include <map> #include <lib/shared/mss_const.H> #include <generic/memory/lib/utils/memory_size.H> +#include <chips/ocmb/explorer/procedures/hwp/memory/exp_inband.H> ///---------------------------------------------------------------------------- /// Constant definitions @@ -100,17 +104,22 @@ static const struct groupSizeTable_t } GROUP_SIZE_TABLE[] = { // GroupSize Encoded GroupSize - { 4, 0b00000000000 }, // 4 GB - { 8, 0b00000000001 }, // 8 GB - { 16, 0b00000000011 }, // 16 GB - { 32, 0b00000000111 }, // 32 GB - { 64, 0b00000001111 }, // 64 GB - { 128, 0b00000011111 }, // 128 GB - { 256, 0b00000111111 }, // 256 GB - { 512, 0b00001111111 }, // 512 GB - { 1024, 0b00011111111 }, // 1 TB - { 2048, 0b00111111111 }, // 2 TB - { 4096, 0b01111111111 }, // 4 TB + { 4, 0b00000000000 }, // 4 GB + { 8, 0b00000000001 }, // 8 GB + { 16, 0b00000000011 }, // 16 GB + { 32, 0b00000000111 }, // 32 GB + { 64, 0b00000001111 }, // 64 GB + { 128, 0b00000011111 }, // 128 GB + { 256, 0b00000111111 }, // 256 GB + { 512, 0b00001111111 }, // 512 GB + { 1024, 0b00011111111 }, // 1 TB + { 2048, 0b00111111111 }, // 2 TB + { 4096, 0b01111111111 }, // 4 TB + { 8192, 0b000011111111111 }, // 8 TB + { 16384, 0b000111111111111 }, // 16 TB + { 32768, 0b001111111111111 }, // 32 TB + { 65536, 0b011111111111111 }, // 64 TB + { 131072, 0b111111111111111 } // 128 TB }; /** @@ -318,7 +327,7 @@ fapi2::ReturnCode getMcMemSize( // Figure out the amount of memory behind this MI // by adding up all memory from its DMI ports auto l_dmiChiplets = i_target.getChildren<fapi2::TARGET_TYPE_DMI>(); - uint64_t l_dmiSize = 0; + uint64_t l_chSize = 0; for (auto l_dmi : l_dmiChiplets) { @@ -328,12 +337,12 @@ fapi2::ReturnCode getMcMemSize( (uint64_t)fapi2::current_err); // Get the amount of memory behind this DMI target - FAPI_TRY(mss::eff_memory_size<mss::mc_type::CENTAUR>(l_dmi, l_dmiSize), + FAPI_TRY(mss::eff_memory_size<mss::mc_type::CENTAUR>(l_dmi, l_chSize), "Error returned from eff_memory_size - DMI, l_rc 0x%.8X", (uint64_t)fapi2::current_err); - FAPI_INF("DMI %u: Total DIMM size %lu GB", l_dmiPos, l_dmiSize); - o_mcSize += l_dmiSize; + FAPI_INF("DMI %u: Total DIMM size %lu GB", l_dmiPos, l_chSize); + o_mcSize += l_chSize; } fapi_try_exit: @@ -341,6 +350,64 @@ fapi_try_exit: return fapi2::current_err; } +/// MC = MCC (Axone) +template<> +fapi2::ReturnCode getMcMemSize( + const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_target, + uint64_t& o_mcSize) +{ + FAPI_DBG("Entering"); + + // Figure out the amount of memory behind this MI + // by adding up all memory from its OMI ports + uint64_t l_chSize = 0; + uint64_t l_sub_size[SUBCHANNEL_PER_CHANNEL]; + uint64_t l_sub_minsize = 0; + uint64_t l_num_sub = 0; + + auto l_omiChiplets = i_target.getChildren<fapi2::TARGET_TYPE_OMI>(); + memset(l_sub_size, 0, sizeof(l_sub_size)); + + for (auto l_omi : l_omiChiplets) + { + const auto& l_ocmb_chiplets = l_omi.getChildren<fapi2::TARGET_TYPE_OCMB_CHIP>(); + + if (!l_ocmb_chiplets.empty()) + { + uint8_t l_omiPos = 0; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_omi, l_omiPos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Get the amount of memory behind this OMI + FAPI_TRY(mss::eff_memory_size<mss::mc_type::EXPLORER>(l_ocmb_chiplets[0], l_chSize), + "Error returned from eff_memory_size - ocmb, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + FAPI_INF("OMI %u: Total DIMM size %lu GB", l_omiPos, l_chSize); + + l_sub_size[l_omiPos % SUBCHANNEL_PER_CHANNEL] = l_chSize; + + if (l_chSize > 0) + { + l_num_sub++; + + if (l_sub_minsize == 0 || l_sub_minsize > l_chSize) + { + l_sub_minsize = l_chSize; + } + } + } + } + + l_chSize = (l_num_sub * l_sub_minsize); + o_mcSize += l_chSize; + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + //////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// /// @@ -348,6 +415,7 @@ fapi_try_exit: /// from group data. /// /// @param[in] i_mcPos MC position +/// @param[in] i_omi Chip uses OMI? /// @param[in] i_groupData Array of Group data info /// @param[out] o_portFound Mark how many time a port is found. /// @param[out] o_mcSize The total mem size calculated @@ -356,6 +424,7 @@ fapi_try_exit: /// void getGroupDataMcMemSize( uint8_t i_mcPos, + const bool i_omi, const uint32_t i_groupData[DATA_GROUPS][DATA_ELEMENTS], uint8_t o_portFound[NUM_MC_PORTS_PER_PROC], uint64_t& o_mcSize) @@ -381,7 +450,8 @@ void getGroupDataMcMemSize( uint8_t l_mcId = getMCPosition(i_groupData[l_group][MEMBER_IDX(0) + l_memberIdx]); - if (l_mcId == i_mcPos) + if ((!i_omi && l_mcId == i_mcPos) || + (i_omi && i_mcPos == i_groupData[l_group][MEMBER_IDX(0) + l_memberIdx])) { o_mcSize += i_groupData[l_group][PORT_SIZE]; FAPI_INF("getGroupDataMcMemSize - Port %u, DIMM size %lu GB", @@ -409,6 +479,7 @@ void getGroupDataMcMemSize( /// - An MCA/DMMI port can only appear once in any group. /// /// @param[in] i_mcTargets Vector of reference of MC targets (MCS or MI) +/// @param[in] i_omi Chip uses OMI? /// @param[in] i_groupData Array of Group data info /// /// @return FAPI2_RC_SUCCESS if success, else error code. @@ -416,6 +487,7 @@ void getGroupDataMcMemSize( template<fapi2::TargetType T> fapi2::ReturnCode validateGroupData( const std::vector< fapi2::Target<T> >& i_mcTargets, + const bool i_omi, const uint32_t i_groupData[DATA_GROUPS][DATA_ELEMENTS]) { FAPI_DBG("Entering"); @@ -444,7 +516,7 @@ fapi2::ReturnCode validateGroupData( (uint64_t)fapi2::current_err); // Get this MC memsize reported in Group data - getGroupDataMcMemSize(l_mcPos, i_groupData, l_portFound, + getGroupDataMcMemSize(l_mcPos, i_omi, i_groupData, l_portFound, l_mcSizeGroupData); FAPI_DBG("validateGroupData: MemSize %.16lld, Group Memsize %.16lld", @@ -655,6 +727,28 @@ fapi2::ReturnCode getNonMirrorBarData(const fapi2::Target<T>& i_mcTarget, } } + FAPI_TRY(getNonMirrorBarIdSize(i_mcTarget, i_portInfo, o_mcBarData)); + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Calculate the BAR data for each MC (MCS/MI) based on group info +/// of port0/1 +/// +/// @param[in] i_mcTarget MC target (MCS/MI) +/// @param[in] i_portInfo The port group info +/// @param[in] o_mcBarData MC BAR data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +template<fapi2::TargetType T> +fapi2::ReturnCode getNonMirrorBarIdSize(const fapi2::Target<T>& i_mcTarget, + const mcPortGroupInfo_t i_portInfo[], + mcBarData_t& o_mcBarData) +{ // MCFGP Channel_0 Group member ID (bits 5:7) o_mcBarData.MCFGP_chan0_group_member_id = i_portInfo[0].channelId; // MCFGP Channel_1 Group member ID (bits 8:10) @@ -756,6 +850,154 @@ fapi_try_exit: } /// +/// @brief Calculate the BAR data for each MC (MCS/MI) based on group info +/// of port0/1 +/// +/// @param[in] i_mcTarget MC target (MCS/MI) +/// @param[in] i_portInfo The port group info +/// @param[in] o_mcBarData MC BAR data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode getNonMirrorBarIdSize(const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_mcTarget, + const mcPortGroupInfo_t& i_portInfo, + mcBarData_t& o_mcBarData) +{ + // MCFGP Channel_0 Group member ID (bits 5:7) + o_mcBarData.MCFGP_chan0_group_member_id = i_portInfo.channelId; + + // If MCFGP is valid, set other fields + if (o_mcBarData.MCFGP_valid == true) + { + // MCFGP Group size + FAPI_TRY(getGroupSizeEncodedValue(i_mcTarget, i_portInfo.groupSize, + o_mcBarData.MCFGP_group_size), + "getGroupSizeEncodedValue() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Group base address + o_mcBarData.MCFGP_groupBaseAddr = i_portInfo.groupBaseAddr; + } + + // If MCFGPM is valid, set other fields + if (o_mcBarData.MCFGPM_valid == true) + { + // MCFGPM Group size + FAPI_TRY(getGroupSizeEncodedValue(i_mcTarget, i_portInfo.groupSize, + o_mcBarData.MCFGPM_group_size), + "getGroupSizeEncodedValue() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Group base address + o_mcBarData.MCFGPM_groupBaseAddr = i_portInfo.groupBaseAddr; + } + + // ---------------------------------------------------- + // Determine data for MCFGPA and MCFGPMA registers + // ---------------------------------------------------- + + // Alternate Memory MCFGPA + for (uint8_t ii = 0; ii < NUM_OF_ALT_MEM_REGIONS; ii++) + { + if ( i_portInfo.altMemValid[ii] ) + { + o_mcBarData.MCFGPA_HOLE_valid[ii] = 1; + o_mcBarData.MCFGPA_HOLE_LOWER_addr[ii] = i_portInfo.altBaseAddr[ii]; + o_mcBarData.MCFGPA_HOLE_UPPER_addr[ii] = + i_portInfo.altBaseAddr[ii] + i_portInfo.altMemSize[ii]; + } + else + { + o_mcBarData.MCFGPA_HOLE_valid[ii] = 0; + o_mcBarData.MCFGPA_HOLE_LOWER_addr[ii] = 0; + o_mcBarData.MCFGPA_HOLE_UPPER_addr[ii] = 0; + } + + } + + // SMF Section of MCFGPA and MCFGPMA + if ( i_portInfo.smfMemValid ) + { + o_mcBarData.MCFGPA_SMF_valid = 1; + o_mcBarData.MCFGPA_SMF_LOWER_addr = i_portInfo.smfBaseAddr; + o_mcBarData.MCFGPA_SMF_UPPER_addr = i_portInfo.smfBaseAddr + i_portInfo.smfMemSize; + } + else + { + o_mcBarData.MCFGPA_SMF_valid = 0; + o_mcBarData.MCFGPA_SMF_LOWER_addr = 0; + o_mcBarData.MCFGPA_SMF_UPPER_addr = 0; + } + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Calculate the BAR data for each MCC based on group info +/// of port0/1 - OMI specific +/// +/// @param[in] i_mcTarget MC target (MCC) +/// @param[in] i_portInfo The port group info +/// @param[in] o_mcBarData MC BAR data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode getNonMirrorBarData(const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_mcTarget, + const mcPortGroupInfo_t& i_portInfo, + mcBarData_t& o_mcBarData) +{ + FAPI_DBG("Entering"); + + // Initialize + o_mcBarData.MCFGP_chan_per_group = NO_CHANNEL_PER_GROUP; + o_mcBarData.MCFGP_valid = false; + o_mcBarData.MCFGPM_valid = false; + + // MCFGP valid (MCFGP bit 0) + if ( i_portInfo.numPortsInGroup > 0) + { + o_mcBarData.MCFGP_valid = true; + o_mcBarData.MCFGPM_valid = false; + + // ---------------------------------------------------- + // Determine data for MCFGP and MCFGPM registers + // ---------------------------------------------------- + if (i_portInfo.numPortsInGroup == 8) + { + o_mcBarData.MCFGP_chan_per_group = 0; + } + else if (i_portInfo.numPortsInGroup == 1 || + i_portInfo.numPortsInGroup == 2 || + i_portInfo.numPortsInGroup == 3 || + i_portInfo.numPortsInGroup == 4 || + i_portInfo.numPortsInGroup == 6 ) + { + o_mcBarData.MCFGP_chan_per_group = i_portInfo.numPortsInGroup; + } + + // Assert if ports 0/1 don't match any entry in table + FAPI_ASSERT(o_mcBarData.MCFGP_chan_per_group != NO_CHANNEL_PER_GROUP, + fapi2::MSS_SETUP_BARS_INVALID_PORTS_CONFIG() + .set_MC_TARGET(i_mcTarget) + .set_PORT_0_PORTS_IN_GROUP(i_portInfo.numPortsInGroup) + .set_PORT_0_GROUP(i_portInfo.myGroup) + .set_PORT_1_PORTS_IN_GROUP(0) + .set_PORT_1_GROUP(0), + "Error: Invalid number of ports per group" + "group %u, ports in group %u", + i_portInfo.myGroup, i_portInfo.numPortsInGroup); + } + + FAPI_TRY(getNonMirrorBarIdSize(i_mcTarget, i_portInfo, o_mcBarData)); + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// /// @brief Calculate the mirror BAR data for each MC based on group info /// of port0/1 /// @@ -779,8 +1021,8 @@ fapi2::ReturnCode getMirrorBarData(const fapi2::Target<T>& i_mcTarget, // Check MCFGP value to see if mirror is possible // (See Table 1 of P9 Cumulus Memory Controller Workbook) // - if ( (io_mcBarData.MCFGP_chan_per_group < 0b0101) || - (io_mcBarData.MCFGP_chan_per_group > 0b1000) ) + if ( ((io_mcBarData.MCFGP_chan_per_group < 0b0101) || + (io_mcBarData.MCFGP_chan_per_group > 0b1000))) { FAPI_IMP("Mirror is not possible with MCFGP = 0x%.8X, NO MIRROR is " "programmed. ", io_mcBarData.MCFGP_chan_per_group); @@ -832,39 +1074,121 @@ fapi_try_exit: } /// -/// @brief Display the Memory controller BAR data resulted from the BAR -/// data calculations. +/// @brief Calculate the mirror BAR data for each MC based on group info +/// of port0/1 /// -/// @param[in] i_portInfo Port data +/// @param[in] i_mcTarget MCC target +/// @param[in] i_portInfo The port group info +/// @param[in] io_mcBarData MC BAR data /// /// @return FAPI2_RC_SUCCESS if success, else error code. /// -void displayMCPortInfoData(const mcPortGroupInfo_t i_portInfo[]) +fapi2::ReturnCode getMirrorBarData(const fapi2::Target<fapi2::TARGET_TYPE_MCC>& i_mcTarget, + const mcPortGroupInfo_t& i_portInfo, + mcBarData_t& io_mcBarData) { - for (uint8_t ii = 0; ii < MAX_MC_PORTS_PER_MCS; ii++) + FAPI_DBG("Entering"); + + // --------------------------------------------------- + // Build MC register values for mirror groups + // --------------------------------------------------- + + // Set MCFGPM_VALID + if (i_portInfo.groupSize > 0) { - FAPI_INF(" Port %u:", ii); + io_mcBarData.MCFGPM_valid = true; - if (i_portInfo[ii].numPortsInGroup > 0) - { - FAPI_INF(" myGroup %u", i_portInfo[ii].myGroup); - FAPI_INF(" numPortsInGroup %u", i_portInfo[ii].numPortsInGroup); - FAPI_INF(" groupSize %u", i_portInfo[ii].groupSize); - FAPI_INF(" groupBaseAddr 0x%.16llX", i_portInfo[ii].groupBaseAddr); - FAPI_INF(" channelId %u", i_portInfo[ii].channelId); + // ---------------------------------------------------- + // Determine data for MCFGPM register + // ---------------------------------------------------- + + // MCFGPM Group size + FAPI_TRY(getGroupSizeEncodedValue(i_mcTarget, i_portInfo.groupSize, + io_mcBarData.MCFGPM_group_size), + "getGroupSizeEncodedValue() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Group base address + io_mcBarData.MCFGPM_groupBaseAddr = i_portInfo.groupBaseAddr; - for (uint8_t jj = 0; jj < NUM_OF_ALT_MEM_REGIONS; jj++) + // ---------------------------------------------------- + // Determine data for MCFGPMA registers + // ---------------------------------------------------- + // Alternate Memory MCFGPMA + for (uint8_t ii = 0; ii < NUM_OF_ALT_MEM_REGIONS; ii++) + { + if ( i_portInfo.altMemValid[ii] ) + { + io_mcBarData.MCFGPMA_HOLE_valid[ii] = 1; + io_mcBarData.MCFGPMA_HOLE_LOWER_addr[ii] = + i_portInfo.altBaseAddr[ii]; + io_mcBarData.MCFGPMA_HOLE_UPPER_addr[ii] = + i_portInfo.altBaseAddr[ii] + i_portInfo.altMemSize[ii]; + } + else { - FAPI_INF(" altMemValid[%u] %u", jj, i_portInfo[ii].altMemValid[jj]); - FAPI_INF(" altMemSize[%u] %u", jj, i_portInfo[ii].altMemSize[jj]); - FAPI_INF(" altBaseAddr[%u] 0x%.16llX", jj, i_portInfo[ii].altBaseAddr[jj]); + io_mcBarData.MCFGPMA_HOLE_valid[ii] = 0; + io_mcBarData.MCFGPMA_HOLE_LOWER_addr[ii] = 0; + io_mcBarData.MCFGPMA_HOLE_UPPER_addr[ii] = 0; } + } - else + } + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Display the Memory controller BAR data resulted from the BAR +/// data calculations. +/// +/// @param[in] i_portInfo Port data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +void displayMCPortInfoData(const mcPortGroupInfo_t i_portInfo) +{ + if (i_portInfo.numPortsInGroup > 0) + { + FAPI_INF(" myGroup %u", i_portInfo.myGroup); + FAPI_INF(" numPortsInGroup %u", i_portInfo.numPortsInGroup); + FAPI_INF(" groupSize %u", i_portInfo.groupSize); + FAPI_INF(" groupBaseAddr 0x%.16llX", i_portInfo.groupBaseAddr); + FAPI_INF(" channelId %u", i_portInfo.channelId); + + for (uint8_t jj = 0; jj < NUM_OF_ALT_MEM_REGIONS; jj++) { - FAPI_INF(" Not configured"); + FAPI_INF(" altMemValid[%u] %u", jj, i_portInfo.altMemValid[jj]); + FAPI_INF(" altMemSize[%u] %u", jj, i_portInfo.altMemSize[jj]); + FAPI_INF(" altBaseAddr[%u] 0x%.16llX", jj, i_portInfo.altBaseAddr[jj]); } } + else + { + FAPI_INF(" Not configured"); + } + + return; +} + +/// +/// @brief Display the Memory controller BAR data resulted from the BAR +/// data calculations. +/// +/// @param[in] i_portInfo Port data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +void displayMCPortInfoData(const mcPortGroupInfo_t i_portInfo[]) +{ + for (uint8_t ii = 0; ii < MAX_MC_PORTS_PER_MCS; ii++) + { + FAPI_INF(" Port %u:", ii); + + displayMCPortInfoData(i_portInfo[ii]); + } return; } @@ -949,6 +1273,7 @@ void getPortData(const bool i_nonMirror, // Skip empty groups if (i_groupData[l_group][GROUP_SIZE] == 0) { + FAPI_DBG("Skipping zero group %d (nonMirror: %d)", l_group, i_nonMirror); continue; } @@ -1022,6 +1347,95 @@ void getPortData(const bool i_nonMirror, } /// +/// @brief Load the mcPortGroupInfo_t data for the input MC based on +/// MC position, input group data, and mirror/non-mirror setting. +/// +/// @param[in] i_nonMirror Type of group data: +/// true = non-mirror; false = mirrored +/// @param[in] i_mcPos MCC position +/// @param[in] i_groupData Array of Group data info +/// @param[out] o_portInfo Output mcPortGroupInfo_t +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +void getPortData(const bool i_nonMirror, + const uint8_t i_mccPos, + const uint32_t i_groupData[DATA_GROUPS][DATA_ELEMENTS], + mcPortGroupInfo_t& o_portInfo) +{ + FAPI_DBG("Entering"); + + // Non-mirrored groups: 0->7 + // Mirrored groups: 8->11 + uint8_t l_startGroup = 0; + uint8_t l_endGroup = (DATA_GROUPS / 2); + + if (i_nonMirror == false) + { + l_startGroup = MIRR_OFFSET; + l_endGroup = (MIRR_OFFSET + NUM_MIRROR_REGIONS); + } + + // Loop thru specified groups + for (uint8_t l_group = l_startGroup; l_group < l_endGroup; l_group++) + { + // Skip empty groups + if (i_groupData[l_group][GROUP_SIZE] == 0) + { + continue; + } + + // Loop thru the ports (MCA/DMI) and determine if they belong + // to this MC (MCS/MI) + for (uint8_t l_memberIdx = 0; + l_memberIdx < i_groupData[l_group][PORTS_IN_GROUP]; l_memberIdx++) + { + uint8_t l_mccPos = i_groupData[l_group][MEMBER_IDX(0) + l_memberIdx]; + + // If the PORT_ID belongs to this MC + if (l_mccPos == i_mccPos) + { + // Set the port group info for this port + o_portInfo.myGroup = l_group; + o_portInfo.numPortsInGroup = i_groupData[l_group][PORTS_IN_GROUP]; + o_portInfo.groupSize = i_groupData[l_group][GROUP_SIZE]; + o_portInfo.groupBaseAddr = i_groupData[l_group][BASE_ADDR]; + o_portInfo.channelId = l_memberIdx; + + // ALT memory regions + for (uint8_t ii = 0; ii < NUM_OF_ALT_MEM_REGIONS; ii++) + { + if (i_groupData[l_group][ALT_VALID(ii)]) + { + o_portInfo.altMemValid[ii] = 1; + o_portInfo.altMemSize[ii] = i_groupData[l_group][ALT_SIZE(ii)]; + o_portInfo.altBaseAddr[ii] = i_groupData[l_group][ALT_BASE_ADDR(ii)]; + } + } + + // SMF memory regions + if (i_groupData[l_group][SMF_VALID]) + { + o_portInfo.smfMemValid = 1; + o_portInfo.smfMemSize = i_groupData[l_group][SMF_SIZE]; + o_portInfo.smfBaseAddr = i_groupData[l_group][SMF_BASE_ADDR]; + } + } + + } // Port loop + + } // Group loop + + // Display MC port info data + FAPI_INF("getPortData from %s group - Results for MCC pos %d", + i_nonMirror ? "NON-MIRROR" : "MIRROR", i_mccPos); + displayMCPortInfoData(o_portInfo); + + FAPI_DBG("Exit"); + return; +} + +/// /// @brief Use Group data obtained from p9_mss_eff_grouping to build /// data to be programmed into each Memory controller target (MCS /// or MI). @@ -1123,6 +1537,101 @@ fapi_try_exit: } /// +/// @brief Use Group data obtained from p9_mss_eff_grouping to build +/// data to be programmed into each Memory controller target MCC +/// for OMI based memory +/// +/// @param[in] i_mcTargets Vector of reference of MCC +/// @param[in] i_groupData Array of Group data info +/// @param[out] o_mcDataPair Output data pair MCC<->Data +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +template<> +fapi2::ReturnCode buildMCBarData( + const std::vector< fapi2::Target<fapi2::TARGET_TYPE_MCC> >& i_mccTargets, + const uint32_t i_groupData[DATA_GROUPS][DATA_ELEMENTS], + std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MCC>, mcBarData_t>>& o_mcBarDataPair) +{ + FAPI_DBG("Entering"); + + char l_targetStr[fapi2::MAX_ECMD_STRING_LEN]; + const fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> FAPI_SYSTEM; + fapi2::ATTR_MRW_HW_MIRRORING_ENABLE_Type l_mirror_ctl; + + // Get mirror policy + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_MRW_HW_MIRRORING_ENABLE, + FAPI_SYSTEM, l_mirror_ctl), + "Error getting ATTR_MRW_HW_MIRRORING_ENABLE, " + "l_rc 0x%.8X", (uint64_t)fapi2::current_err); + + for (auto l_mcc : i_mccTargets) + { + mcBarData_t l_mcBarData; + mcPortGroupInfo_t l_portInfo; + + // Get this MC unit position + uint8_t l_unitPos = 0; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mcc, l_unitPos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + fapi2::toString(l_mcc, l_targetStr, sizeof(l_targetStr)); + FAPI_INF("Build BAR data for MC target: %s", l_targetStr); + + // ----------------------------------------------- + // Build MC register values for non-mirror groups + // ----------------------------------------------- + + // Get port data from non-mirrored groups (true = non-mirrored) + getPortData(true, l_unitPos, i_groupData, l_portInfo); + + // If one of MC port is configured in a group, proceed with + // getting BAR data + if ( (l_portInfo.numPortsInGroup > 0) ) + { + // ---- Build MCFGP/MCFGM data based on port group info ---- + FAPI_TRY(getNonMirrorBarData(l_mcc, l_portInfo, l_mcBarData), + "getNonMirrorBarData() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // --------------------------------------------------------------- + // Set MC register values for mirror groups + // - Nimbus: No mirror + // - Cumulus: If ATTR_MRW_HW_MIRRORING_ENABLE != false + // --------------------------------------------------------------- + if (l_mirror_ctl != fapi2::ENUM_ATTR_MRW_HW_MIRRORING_ENABLE_FALSE) + { + FAPI_INF("ATTR_MRW_HW_MIRRORING_ENABLE is enabled: checking mirrored groups"); + mcPortGroupInfo_t l_portInfoMirrored; + // Get port data from mirrored groups (false = mirrored) + getPortData(false, l_unitPos, i_groupData, l_portInfoMirrored); + + // ---- Build MCFGM data based on port group info ---- + FAPI_TRY(getMirrorBarData(l_mcc, l_portInfoMirrored, l_mcBarData), + "getMirrorBarData() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + } + + // Add to output pair + o_mcBarDataPair.push_back(std::make_pair(l_mcc, l_mcBarData)); + + // Display data + displayMCBarData(l_unitPos, l_mcBarData); + } + else + { + FAPI_INF("MC pos %u is not configured in a memory group.", l_unitPos); + } + + } // MC loop + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// /// @brief Set MCFGPA MEMORY HOLE UPPER_ADDRESS_AT_END_OF_RANGE bit /// /// @param[in] i_target MC target (MCS or MI) @@ -1196,6 +1705,7 @@ fapi2::ReturnCode writeMCBarData( MCS_MCFGP_GROUP_SIZE_LEN>( l_data.MCFGP_group_size); + // Group base address (bits 24:47) 0b000000000000000000000001 = 4GB // 000000001 (base addr of 4GB) // 000000010 (base addr of 8GB) @@ -1443,15 +1953,378 @@ fapi_try_exit: } /// -/// @brief Unmask FIR before opening BARs +/// @brief Write BAR data to a memory controller +/// +/// @param[in] i_mcBarDataPair Target pair <target, data> +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode writeMCCInterleaveGranularity( + const std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MCC>, mcBarData_t>>& i_mcBarDataPair) +{ + FAPI_DBG("Entering"); + + std::map<fapi2::Target<fapi2::TARGET_TYPE_MI>, bool> l_granule_supported; + + fapi2::ATTR_MSS_INTERLEAVE_GRANULARITY_Type l_interleave_granule_size; + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_MSS_INTERLEAVE_GRANULARITY, + fapi2::Target<fapi2::TARGET_TYPE_SYSTEM>(), + l_interleave_granule_size), + "Error getting ATTR_MSS_INTERLEAVE_GRANULARITY, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + for (auto l_pair : i_mcBarDataPair) + { + fapi2::Target<fapi2::TARGET_TYPE_MCC> l_target = l_pair.first; + mcBarData_t l_data = l_pair.second; + + // MCFGP valid (bit 0) + if (l_data.MCFGP_valid == true) + { + fapi2::Target<fapi2::TARGET_TYPE_MI> l_mi_target = l_target.getParent<fapi2::TARGET_TYPE_MI>(); + + // configure interleave granularity if 1/2/4/8 MC per group only + if ((l_data.MCFGP_chan_per_group == 0) || // 8 MC/group + (l_data.MCFGP_chan_per_group == 1) || // 1 MC/group + (l_data.MCFGP_chan_per_group == 2) || // 2 MC/group + (l_data.MCFGP_chan_per_group == 4)) // 4 MC/group + { + //Only set to true if the value for the target is not set yet + if (l_granule_supported.find(l_mi_target) == l_granule_supported.end()) + { + l_granule_supported[l_mi_target] = true; + } + } + else + { + //Always set to false if we find one channel that cannot support it. + l_granule_supported[l_mi_target] = false; + } + } + } + + for ( auto l_it = l_granule_supported.begin(); l_it != l_granule_supported.end(); l_it++ ) + { + if (l_it->second) + { + auto l_mi_target = l_it->first; + + fapi2::buffer<uint64_t> l_mcmode0_scom_data; + FAPI_TRY(fapi2::getScom(l_mi_target, P9A_MI_MCMODE0, l_mcmode0_scom_data), + "Error reading from MCS_MCMODE0 reg"); + l_mcmode0_scom_data.insertFromRight<P9A_MI_MCMODE0_GROUP_ADDRESS_INTERLEAVE_GRANULARITY, + P9A_MI_MCMODE0_GROUP_ADDRESS_INTERLEAVE_GRANULARITY_LEN>(l_interleave_granule_size); + FAPI_TRY(fapi2::putScom(l_mi_target, P9A_MI_MCMODE0, l_mcmode0_scom_data), + "Error writing to MCS_MCMODE0 reg"); + } + } + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Translate a bar value in GB to a 32 bit extended value +/// +/// @param[in] i_ext_mask The extension mask +/// @param[in] i_bar The bar in GB +/// @param[out] o_extBar is the upper 32 bits of the extended address +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +fapi2::ReturnCode extBar(const uint64_t& i_ext_mask, const uint32_t& i_bar, fapi2::buffer<uint64_t>& o_extBar) +{ + FAPI_DBG("Entering"); + fapi2::buffer<uint64_t> l_norAddr(0); + + FAPI_DBG("i_bar: %016llx", i_bar); + o_extBar = 0; + l_norAddr.insert<0, 32>(i_bar >> 2); //i_bar is in GB + FAPI_DBG("l_norAddr: %016llx", l_norAddr); + FAPI_TRY(extendBarAddress(i_ext_mask, l_norAddr, o_extBar)); + FAPI_DBG("o_extBar: %016llx", o_extBar); + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Write BAR data to a memory controller /// /// @param[in] i_mcBarDataPair Target pair <target, data> /// /// @return FAPI2_RC_SUCCESS if success, else error code. /// +fapi2::ReturnCode writeMCBarData( + const std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MCC>, mcBarData_t>>& i_mcBarDataPair) +{ + FAPI_DBG("Entering"); + + uint8_t l_pos; + + fapi2::buffer<uint64_t> l_scomData(0); + fapi2::buffer<uint64_t> l_extAddr(0); + fapi2::buffer<uint64_t> l_norAddr; + uint64_t l_ext_mask; + + FAPI_TRY(p9a_get_ext_mask(l_ext_mask)); + + FAPI_TRY(writeMCCInterleaveGranularity(i_mcBarDataPair)); + + for (auto l_pair : i_mcBarDataPair) + { + fapi2::Target<fapi2::TARGET_TYPE_MCC> l_target = l_pair.first; + mcBarData_t l_data = l_pair.second; + + char l_targetStr[fapi2::MAX_ECMD_STRING_LEN]; + fapi2::toString(l_target, l_targetStr, sizeof(l_targetStr)); + FAPI_INF("Program MC target: %s", l_targetStr); + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_target, l_pos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // 1. ---- Set MCFGP reg ----- + l_scomData = 0; + + // MCFGP valid (bit 0) + if (l_data.MCFGP_valid == true) + { + l_scomData.setBit<P9A_MI_MCFGP0_VALID>(); + + // Group size (bits 13:23) + l_scomData.insertFromRight<P9A_MI_MCFGP0_GROUP_SIZE, + P9A_MI_MCFGP0_GROUP_SIZE_LEN>( + l_data.MCFGP_group_size); + + FAPI_TRY(extBar(l_ext_mask, l_data.MCFGP_groupBaseAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGP0_GROUP_BASE_ADDRESS, + P9A_MI_MCFGP0_GROUP_BASE_ADDRESS_LEN>( + (l_extAddr)); + } + + // Channel per group (bits 1:4) + l_scomData.insertFromRight<P9A_MI_MCFGP0_MC_CHANNELS_PER_GROUP, + P9A_MI_MCFGP0_MC_CHANNELS_PER_GROUP_LEN>( + l_data.MCFGP_chan_per_group); + + // Channel 0 group id (bits 5:7) + l_scomData.insertFromRight<P9A_MI_MCFGP0_GROUP_MEMBER_IDENTIFICATION, + P9A_MI_MCFGP0_GROUP_MEMBER_IDENTIFICATION_LEN>( + l_data.MCFGP_chan0_group_member_id); + + // 2GB cfg and MMIO + l_scomData.insertFromRight<P9A_MI_MCFGP0_MCFGPR0_CONFIGURATION_GROUP_SIZE, + P9A_MI_MCFGP0_MCFGPR0_CONFIGURATION_GROUP_SIZE_LEN>(mss::exp::ib::EXPLR_IB_BAR_SIZE); + l_scomData.insertFromRight<P9A_MI_MCFGP0_MCFGPR0_MMIO_GROUP_SIZE, + P9A_MI_MCFGP0_MCFGPR0_MMIO_GROUP_SIZE_LEN>(mss::exp::ib::EXPLR_IB_BAR_SIZE); + + // Write to reg + if (l_pos % 2 == 0) + { + FAPI_INF("Write MCFGP0 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP0, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP0, l_scomData), + "Error writing to MCS_MCFGP reg"); + } + else + { + FAPI_INF("Write MCFGP1 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP1, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP1, l_scomData), + "Error writing to MCS_MCFGP reg"); + } + + // 2. ---- Set MCFGPM reg ----- + l_scomData = 0; + + if (l_data.MCFGPM_valid == true) + { + // MCFGP valid (bit 0) + l_scomData.setBit<P9A_MI_MCFGPM0_VALID>(); + + // Group size (bits 13:23) + l_scomData.insertFromRight<P9A_MI_MCFGPM0_GROUP_SIZE, + P9A_MI_MCFGPM0_GROUP_SIZE_LEN>( + l_data.MCFGPM_group_size); + + FAPI_TRY(extBar(l_ext_mask, l_data.MCFGPM_groupBaseAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGPM0_GROUP_BASE_ADDRESS, + P9A_MI_MCFGPM0_GROUP_BASE_ADDRESS_LEN>( + (l_extAddr)); + + } + + if (l_pos % 2 == 0) + { + // Write to reg + FAPI_INF("Write MCFGPM0 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGPM0, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGPM0, l_scomData), + "Error writing to P9A_MI_MCFGPM0 reg"); + } + else + { + // Write to reg + FAPI_INF("Write MCFGPM1 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGPM1, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGPM1, l_scomData), + "Error writing to P9A_MI_MCFGPM1 reg"); + } + + // 3. ---- Set MCFGPA reg ----- + l_scomData = 0; + + // Assert if both HOLE1 and SMF are valid, settings will overlap + FAPI_ASSERT((l_data.MCFGPA_HOLE_valid[1] && l_data.MCFGPA_SMF_valid) == 0, + fapi2::MSS_SETUP_BARS_HOLE1_SMF_CONFLICT() + .set_TARGET(l_target) + .set_HOLE1_VALID(l_data.MCFGPA_HOLE_valid[1]) + .set_SMF_VALID(l_data.MCFGPA_SMF_valid), + "Error: MCFGPA HOLE1 and SMF are both valid, settings will overlap"); + + // Hole 0 + if (l_data.MCFGPA_HOLE_valid[0] == true) + { + // MCFGPA HOLE0 valid (bit 0) + l_scomData.setBit<P9A_MI_MCFGP0A_HOLE_VALID>(); + + // Hole 0 lower addr + // Hole 0 always extends to end of range + FAPI_DBG("l_data.MCFGPA_HOLE_LOWER_addr[0]: %016llx", l_data.MCFGPA_HOLE_LOWER_addr[0]); + FAPI_TRY(extBar(l_ext_mask, l_data.MCFGPA_HOLE_LOWER_addr[0], l_extAddr)); + l_scomData.insert<P9A_MI_MCFGP0A_HOLE_LOWER_ADDRESS, + P9A_MI_MCFGP0A_HOLE_LOWER_ADDRESS_LEN>( + (l_extAddr << 9)); //matches 17:31 extendedBarAddress shifts left 8 (17-8) = 9 + } + + // SMF + if (l_data.MCFGPA_SMF_valid == true) + { + // MCFGPA SMF valid (bit 0) + l_scomData.setBit<P9A_MI_MCFGP0A_SMF_VALID>(); + + // MCFGPA_SMF_UPPER_ADDRESS_AT_END_OF_RANGE + l_scomData.setBit<P9A_MI_MCFGP0A_SMF_EXTEND_TO_END_OF_RANGE>(); + + // SMF lower addr + l_norAddr = 0; + l_norAddr.insertFromRight<22, 14>(l_data.MCFGPA_SMF_LOWER_addr); + FAPI_TRY(extendBarAddress(l_ext_mask, l_norAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGP0A_SMF_LOWER_ADDRESS, + P9A_MI_MCFGP0A_SMF_LOWER_ADDRESS_LEN>( + (l_extAddr << 14)); //matches 22:35 extendBarAddress shifts left 8 (22-8) = 14 + // SMF upper addr + l_norAddr = 0; + l_norAddr.insertFromRight<22, 14>(l_data.MCFGPA_SMF_UPPER_addr); + FAPI_TRY(extendBarAddress(l_ext_mask, l_norAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGP0A_SMF_UPPER_ADDRESS, + P9A_MI_MCFGP0A_SMF_UPPER_ADDRESS_LEN>( + (l_extAddr << 14)); //matches 22:35 extendBarAddress shifts left 8 (22-8) = 14 + } + + // Write to reg + if (l_pos % 2 == 0) + { + FAPI_INF("Write MCFGP0A reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP0A, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP0A, l_scomData), + "Error writing to P9A_MI_MCFGP0A reg"); + } + else + { + FAPI_INF("Write MCFGP1A reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP1A, l_scomData); + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP1A, l_scomData), + "Error writing to P9A_MI_MCFGP1A reg"); + } + + // 4. ---- Set MCFGPMA reg ----- + l_scomData = 0; + + // Assert if both HOLE1 and SMF are valid, settings will overlap + FAPI_ASSERT((l_data.MCFGPMA_HOLE_valid[1] && l_data.MCFGPMA_SMF_valid) == 0, + fapi2::MSS_SETUP_BARS_HOLE1_SMF_CONFLICT() + .set_TARGET(l_target) + .set_HOLE1_VALID(l_data.MCFGPMA_HOLE_valid[1]) + .set_SMF_VALID(l_data.MCFGPMA_SMF_valid), + "Error: MCFGPMA HOLE1 and SMF are both valid, settings will overlap"); + + // Hole 0 + if (l_data.MCFGPMA_HOLE_valid[0] == true) + { + // MCFGPMA HOLE0 valid (bit 0) + l_scomData.setBit<P9A_MI_MCFGPM0A_HOLE_VALID>(); + + // Hole 0 lower addr + // 0b0000000001 = 4GB + FAPI_TRY(extBar(l_ext_mask, l_data.MCFGPMA_HOLE_LOWER_addr[0], l_extAddr)); + l_scomData.insert<P9A_MI_MCFGPM0A_HOLE_LOWER_ADDRESS, + P9A_MI_MCFGPM0A_HOLE_LOWER_ADDRESS_LEN>( + (l_extAddr << 9)); //matches 17:31 extendedBarAddress shifts left 8 (17-8) = 9 + } + + // SMF + if (l_data.MCFGPMA_SMF_valid == true) + { + // MCFGPMA SMF valid (bit 0) + l_scomData.setBit<P9A_MI_MCFGPM0A_SMF_VALID>(); + + // MCFGPMA_SMF_UPPER_ADDRESS_AT_END_OF_RANGE + l_scomData.setBit<P9A_MI_MCFGPM0A_SMF_EXTEND_TO_END_OF_RANGE>(); + + // SMF lower addr + l_norAddr = 0; + l_norAddr.insertFromRight<22, 14>(l_data.MCFGPMA_SMF_LOWER_addr); + FAPI_TRY(extendBarAddress(l_ext_mask, l_norAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGPM0A_SMF_LOWER_ADDRESS, + P9A_MI_MCFGPM0A_SMF_LOWER_ADDRESS_LEN>( + (l_extAddr << 14 )); //matches 22:35 extendBarAddress shifts left 8 (22-8) = 14 + // SMF upper addr + l_norAddr = 0; + l_norAddr.insertFromRight<22, 14>(l_data.MCFGPMA_SMF_UPPER_addr); + FAPI_TRY(extendBarAddress(l_ext_mask, l_norAddr, l_extAddr)); + l_scomData.insert<P9A_MI_MCFGPM0A_SMF_UPPER_ADDRESS, + P9A_MI_MCFGPM0A_SMF_UPPER_ADDRESS_LEN>( + (l_extAddr << 14)); //matches 22:35 extendBarAddress shifts left 8 (22-8) = 14 + } + + // Write to reg + if (l_pos % 2 == 0) + { + FAPI_INF("Write P9A_MI_MCFGPM0A reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGPM0A, l_scomData); + + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGPM0A, l_scomData), + "Error writing to P9A_MI_MCFGPM0A reg"); + } + else + { + FAPI_INF("Write P9A_MI_MCFGPM1A reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGPM1A, l_scomData); + + FAPI_TRY(fapi2::putScom(l_target.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGPM1A, l_scomData), + "Error writing to P9A_MI_MCFGPM1A reg"); + } + + } // Data pair loop + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Unmask FIR before opening BARs +/// +/// @param[in] i_target target to set actions/mask +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// template<fapi2::TargetType T> -fapi2::ReturnCode unmaskMCFIR( - const std::vector<std::pair<fapi2::Target<T>, mcBarData_t>>& i_mcBarDataPair) +fapi2::ReturnCode unmaskMCFIR(const fapi2::Target<T> i_target) { FAPI_DBG("Entering"); @@ -1495,20 +2368,40 @@ fapi2::ReturnCode unmaskMCFIR( // Defect HW451708, HW451711 // Leave MCS_MCFIR_INVALID_SMF_ACCESS masked if MCD cl_probes are enabled + char l_targetStr[fapi2::MAX_ECMD_STRING_LEN]; + fapi2::toString(i_target, l_targetStr, sizeof(l_targetStr)); + FAPI_INF("Unmask FIR for MC target: %s", l_targetStr); + + // Write MC FIR action1 + FAPI_TRY(fapi2::putScom(i_target, MCS_MCFIRACT1, l_mcfiraction), + "Error from putScom (MCS_MCFIRACT1)"); + + // Write mask + FAPI_TRY(fapi2::putScom(i_target, MCS_MCFIRMASK_AND, l_mcfirmask_and), + "Error from putScom (MCS_MCFIRMASK_AND)"); + +fapi_try_exit: + FAPI_DBG("Exit"); + return fapi2::current_err; +} + +/// +/// @brief Unmask FIR before opening BARs +/// +/// @param[in] i_mcBarDataPair Target pair <target, data> +/// +/// @return FAPI2_RC_SUCCESS if success, else error code. +/// +template<fapi2::TargetType T> +fapi2::ReturnCode unmaskMCFIR( + const std::vector<std::pair<fapi2::Target<T>, mcBarData_t>>& i_mcBarDataPair) +{ + FAPI_DBG("Entering"); + for (auto l_pair : i_mcBarDataPair) { fapi2::Target<T> l_target = l_pair.first; - char l_targetStr[fapi2::MAX_ECMD_STRING_LEN]; - fapi2::toString(l_target, l_targetStr, sizeof(l_targetStr)); - FAPI_INF("Unmask FIR for MC target: %s", l_targetStr); - - // Write MC FIR action1 - FAPI_TRY(fapi2::putScom(l_target, MCS_MCFIRACT1, l_mcfiraction), - "Error from putScom (MCS_MCFIRACT1)"); - - // Write mask - FAPI_TRY(fapi2::putScom(l_target, MCS_MCFIRMASK_AND, l_mcfirmask_and), - "Error from putScom (MCS_MCFIRMASK_AND)"); + FAPI_TRY(unmaskMCFIR(l_target)); } // Data pair loop @@ -1534,14 +2427,25 @@ fapi2::ReturnCode p9_mss_setup_bars( std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MCS>, mcBarData_t>> l_mcsBarDataPair; // std_pair<MI target, MI data> std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MI>, mcBarData_t>> l_miBarDataPair; + // std_pair<MCC target, MCC data> + std::vector<std::pair<fapi2::Target<fapi2::TARGET_TYPE_MCC>, mcBarData_t>> l_mccBarDataPair; + + fapi2::ATTR_CHIP_EC_FEATURE_OMI_Type l_omi; // Get functional MCS chiplets, should be none for Cumulus auto l_mcsChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MCS>(); // Get functional MI chiplets, should be none for Nimbus auto l_miChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MI>(); + // Get MCC chiplets for Axone + auto l_mccChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MCC>(); FAPI_INF("Num of MCS %u; Num of MIs %u", l_mcsChiplets.size(), l_miChiplets.size()); + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_OMI, + i_target, + l_omi), + "Error from FAPI_ATTR_GET (ATTR_CHIP_EC_FEATURE_OMI)"); + if ( (l_mcsChiplets.size() > 0) && (l_miChiplets.size() > 0) ) { FAPI_ASSERT(false, @@ -1564,7 +2468,7 @@ fapi2::ReturnCode p9_mss_setup_bars( if (l_mcsChiplets.size() > 0) { // Validate group data from attributes - FAPI_TRY(validateGroupData(l_mcsChiplets, l_groupData), + FAPI_TRY(validateGroupData(l_mcsChiplets, false, l_groupData), "validateGroupData() returns error, l_rc 0x%.8X", (uint64_t)fapi2::current_err); @@ -1584,11 +2488,38 @@ fapi2::ReturnCode p9_mss_setup_bars( (uint64_t)fapi2::current_err); } + // Setup BAR for Axone + else if (l_mccChiplets.size() > 0) + { + // Validate group data from attributes + FAPI_TRY(validateGroupData(l_mccChiplets, true, l_groupData), + "validateGroupData() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Build MC BAR data based on Group data info + FAPI_TRY(buildMCBarData(l_mccChiplets, l_groupData, l_mccBarDataPair), + "buildMCBarData() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + // Unmask MC FIRs + for (auto l_target : l_miChiplets) + { + FAPI_TRY(unmaskMCFIR(l_target), + "unmaskMCFIR() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + } + + // Write data to MI + FAPI_TRY(writeMCBarData(l_mccBarDataPair), + "writeMCBarData() returns error, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + } // Setup BAR for Cumulus else if (l_miChiplets.size() > 0) { // Validate group data from attributes - FAPI_TRY(validateGroupData(l_miChiplets, l_groupData), + FAPI_TRY(validateGroupData(l_miChiplets, false, l_groupData), "validateGroupData() returns error, l_rc 0x%.8X", (uint64_t)fapi2::current_err); diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.mk b/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.mk index db115d16a..1c01fdbc7 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.mk +++ b/src/import/chips/p9/procedures/hwp/nest/p9_mss_setup_bars.mk @@ -25,7 +25,8 @@ # Include the macros and things for MSS procedures PROCEDURE=p9_mss_setup_bars -$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9/procedures/hwp/memory/) +OBJS+=p9a_addr_ext.o lib$(PROCEDURE)_DEPLIBS+=cen +$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9/procedures/hwp/memory/) $(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)) $(call BUILD_PROCEDURE) diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_query_mssinfo.C b/src/import/chips/p9/procedures/hwp/nest/p9_query_mssinfo.C index ea80302fc..3309552ec 100644 --- a/src/import/chips/p9/procedures/hwp/nest/p9_query_mssinfo.C +++ b/src/import/chips/p9/procedures/hwp/nest/p9_query_mssinfo.C @@ -64,8 +64,8 @@ fapi2::ReturnCode p9_query_mssinfo(const std::vector<fapi2::Target<fapi2::TARGET //vars for attrs uint64_t sizes[8]; uint64_t bases[8]; - uint64_t mirror_sizes[4] = {0, 0, 0, 0}; - uint64_t mirror_bases[4]; + uint64_t mirror_sizes[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + uint64_t mirror_bases[8] = {0, 0, 0, 0, 0, 0, 0, 0}; uint32_t groupID[MBA_GROUP_SIZE][MBA_GROUP_DATA]; uint64_t nhtm_base; uint64_t nhtm_size; @@ -83,6 +83,7 @@ fapi2::ReturnCode p9_query_mssinfo(const std::vector<fapi2::Target<fapi2::TARGET const int ONE_GIG = 30; + FAPI_DBG("i_vect_pu_targets.size(): %d", i_vect_pu_targets.size()); //loop over all elements in the vector for(uint32_t i = 0; i < i_vect_pu_targets.size(); i++) @@ -257,7 +258,7 @@ fapi2::ReturnCode p9_query_mssinfo(const std::vector<fapi2::Target<fapi2::TARGET for(int j = 0; j < 8; j++) { if ((sizes[j] != 0) || - ((j < 4) && (mirror_sizes[j] != 0))) + ((mirror_sizes[j] != 0))) { printf("Group:%d (", j); @@ -284,7 +285,7 @@ fapi2::ReturnCode p9_query_mssinfo(const std::vector<fapi2::Target<fapi2::TARGET } - if ((j < 4) && (mirror_sizes[j] != 0)) + if ((mirror_sizes[j] != 0)) { if (mirror_policy == fapi2::ENUM_ATTR_MEM_MIRROR_PLACEMENT_POLICY_FLIPPED) { diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.C b/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.C new file mode 100644 index 000000000..4f6aeeb07 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.C @@ -0,0 +1,187 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.C $ */ +/* */ +/* OpenPOWER HostBoot Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2018,2019 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9a_addr_ext.C +/// @brief Handle address extension mask for Axone +/// + +// *HWP HWP Owner: Ben Gass bgass@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 3 +// *HWP Consumed by: HB + +//----------------------------------------------------------------------------------- +// Includes +//----------------------------------------------------------------------------------- +#include <p9a_addr_ext.H> + +//----------------------------------------------------------------------------------- +// Function definitions +//----------------------------------------------------------------------------------- + +fapi2::ReturnCode p9a_get_ext_mask(uint64_t& o_ext_mask) +{ + FAPI_DBG("Start"); + uint8_t l_addr_extension_group_id = 0; + uint8_t l_addr_extension_chip_id = 0; + fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> FAPI_SYSTEM; + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID, + FAPI_SYSTEM, + l_addr_extension_group_id), + "Error from FAPI_ATTR_GET (ATTR_FABRIC_ADDR_EXTENSION_GROUP_ID)"); + + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID, + FAPI_SYSTEM, + l_addr_extension_chip_id), + "Error from FAPI_ATTR_GET (ATTR_FABRIC_ADDR_EXTENSION_CHIP_ID"); + + o_ext_mask = ((l_addr_extension_group_id & 0xF) << 4) | (l_addr_extension_chip_id & 0xF); + +fapi_try_exit: + FAPI_DBG("End"); + return fapi2::current_err; +} + + + +/* + * As documented + * BAR Bits 6 8 9 7 10 11 12 13 + * 0x00 15 16 17 18 21 20 19 14 + * 0x04 15 16 17 18 21 20 14 19 + * 0x06 15 16 17 18 21 14 20 19 + * 0x07 15 16 17 18 14 21 20 19 + * 0x80 15 16 17 18 21 20 19 14 + * 0x84 15 16 17 18 21 20 14 19 + * 0x86 15 16 17 18 21 14 20 19 + * 0x87 15 16 17 18 14 21 20 19 + * 0xC0 15 17 18 21 20 19 14 16 + * 0xC4 15 17 18 21 20 14 19 16 + * 0xC6 15 17 18 21 14 20 19 16 + * 0xC7 15 17 18 14 21 20 19 16 + * 0xE0 15 18 21 20 19 14 17 16 + * 0xE4 15 18 21 20 14 19 17 16 + * 0xE6 15 18 21 14 20 19 17 16 + * 0xE7 15 18 14 21 20 19 17 16 + * 0xF0 15 21 20 19 14 18 17 16 + * 0xF4 15 21 20 14 19 18 17 16 + * 0xF6 15 21 14 20 19 18 17 16 + * 0xF7 15 14 21 20 19 18 17 16 + * + * Ordered bar bits + * BAR Bits 5 6 7 8 9 10 11 12 13 14 ... + * 0x00 13 15 18 16 17 21 20 19 14 22 ... + * 0x04 13 15 18 16 17 21 20 14 19 22 ... + * 0x06 13 15 18 16 17 21 14 20 19 22 ... + * 0x07 13 15 18 16 17 14 21 20 19 22 ... + * 0x80 13 15 18 16 17 21 20 19 14 22 ... + * 0x84 13 15 18 16 17 21 20 14 19 22 ... + * 0x86 13 15 18 16 17 21 14 20 19 22 ... + * 0x87 13 15 18 16 17 14 21 20 19 22 ... + * 0xC0 13 15 21 17 18 20 19 14 16 22 ... + * 0xC4 13 15 21 17 18 20 14 19 16 22 ... + * 0xC6 13 15 21 17 18 14 20 19 16 22 ... + * 0xC7 13 15 14 17 18 21 20 19 16 22 ... + * 0xE0 13 15 20 18 21 19 14 17 16 22 ... + * 0xE4 13 15 20 18 21 14 19 17 16 22 ... + * 0xE6 13 15 14 18 21 20 19 17 16 22 ... + * 0xE7 13 15 21 18 14 20 19 17 16 22 ... + * 0xF0 13 15 19 21 20 14 18 17 16 22 ... + * 0xF4 13 15 14 21 20 19 18 17 16 22 ... + * 0xF6 13 15 20 21 14 19 18 17 16 22 ... + * 0xF7 13 15 20 14 21 19 18 17 16 22 ... + * + * + */ +fapi2::ReturnCode extendBarAddress(const uint64_t& i_ext_mask, const fapi2::buffer<uint64_t>& i_bar_addr, + fapi2::buffer<uint64_t>& o_bar) +{ + static const uint64_t BAR_START_BIT = 8; + static const uint64_t BAR_MASK_STATIC_BITS = 0xFC03FFFE00000000ull; //31 bit bar, bits 6:13 reordered + static const uint64_t REORDER_START_BIT = 6; // reordering of bar starts at bit 6 + static const int REORDER_END_BIT = 13; // reordering of bar stops at bit 13 + static const int NUM_EXT_MASKS = 20; // 20 different extension masks are supported + static const uint64_t EXT_MASK_REORDER[][9] = // Workbook table 7 + { + // B 6 7 8 9 10 11 12 13 + { 0x00, 15, 18, 16, 17, 21, 20, 19, 14 }, + { 0x04, 15, 18, 16, 17, 21, 20, 14, 19 }, + { 0x06, 15, 18, 16, 17, 21, 14, 20, 19 }, + { 0x07, 15, 18, 16, 17, 14, 21, 20, 19 }, + { 0x80, 15, 18, 16, 17, 21, 20, 19, 14 }, + { 0x84, 15, 18, 16, 17, 21, 20, 14, 19 }, + { 0x86, 15, 18, 16, 17, 21, 14, 20, 19 }, + { 0x87, 15, 18, 16, 17, 14, 21, 20, 19 }, + { 0xC0, 15, 21, 17, 18, 20, 19, 14, 16 }, + { 0xC4, 15, 21, 17, 18, 20, 14, 19, 16 }, + { 0xC6, 15, 21, 17, 18, 14, 20, 19, 16 }, + { 0xC7, 15, 14, 17, 18, 21, 20, 19, 16 }, + { 0xE0, 15, 20, 18, 21, 19, 14, 17, 16 }, + { 0xE4, 15, 20, 18, 21, 14, 19, 17, 16 }, + { 0xE6, 15, 14, 18, 21, 20, 19, 17, 16 }, + { 0xE7, 15, 21, 18, 14, 20, 19, 17, 16 }, + { 0xF0, 15, 19, 21, 20, 14, 18, 17, 16 }, + { 0xF4, 15, 14, 21, 20, 19, 18, 17, 16 }, + { 0xF6, 15, 20, 21, 14, 19, 18, 17, 16 }, + { 0xF7, 15, 20, 14, 21, 19, 18, 17, 16 } + }; + + uint64_t l_bar = i_bar_addr; + l_bar = i_bar_addr << BAR_START_BIT; + l_bar = l_bar & BAR_MASK_STATIC_BITS; + o_bar = l_bar; + FAPI_DBG("i_bar_addr: %llx o_bar: %llx", i_bar_addr, o_bar); + + for (auto l_i = 0; l_i < NUM_EXT_MASKS; l_i++) + { + if (EXT_MASK_REORDER[l_i][0] == i_ext_mask) + { + int l_ro_idx = 1; + + for (auto l_i2 = REORDER_START_BIT; l_i2 <= REORDER_END_BIT; l_i2++) + { + if (i_bar_addr.getBit(EXT_MASK_REORDER[l_i][l_ro_idx])) + { + o_bar.setBit(l_i2); + } + + l_ro_idx++; + } + + goto fapi_try_exit; + } + } + + FAPI_DBG("o_bar: %llx", o_bar); + FAPI_ASSERT(false, + fapi2::PROC_OMI_SETUP_BARS_INVALID_ADDR_EXT_MASK() + .set_EXT_MASK(i_ext_mask), + "An invalid address extension mask was selected"); +fapi_try_exit: + FAPI_DBG("End"); + return fapi2::current_err; +} diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.H b/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.H new file mode 100644 index 000000000..de85fac47 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.H @@ -0,0 +1,52 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9a_addr_ext.H $ */ +/* */ +/* OpenPOWER HostBoot Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2018,2019 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9a_addr_ext.H +/// @brief Handle address extension mask for Axone +/// + +// *HWP HWP Owner: Ben Gass bgass@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 3 +// *HWP Consumed by: HB + +#ifndef _P9A_ADDR_EXT_H_ +#define _P9A_ADDR_EXT_H_ + +//------------------------------------------------------------------------------ +// Includes +//------------------------------------------------------------------------------ +#include <fapi2.H> + +//----------------------------------------------------------------------------------- +// Function definitions +//----------------------------------------------------------------------------------- + +fapi2::ReturnCode p9a_get_ext_mask(uint64_t& o_ext_mask); + +fapi2::ReturnCode extendBarAddress(const uint64_t& i_ext_mask, const fapi2::buffer<uint64_t>& i_bar_addr, + fapi2::buffer<uint64_t>& o_bar); +#endif diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.C b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.C new file mode 100644 index 000000000..e5efc07c5 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.C @@ -0,0 +1,222 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.C $ */ +/* */ +/* OpenPOWER HostBoot Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2017,2019 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9a_omi_setup_bars.H +/// @brief configure Axone inband address +/// +/// Set the inband base address in the MC +/// driven by attributes representing system memory map. +/// + +// *HWP HWP Owner: Ben Gass bgass@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 3 +// *HWP Consumed by: HB + +//----------------------------------------------------------------------------------- +// Includes +//----------------------------------------------------------------------------------- +#include <p9a_omi_setup_bars.H> +#include <p9a_addr_ext.H> +#include <p9_fbc_utils.H> +#include <p9_mc_scom_addresses.H> +#include <p9a_misc_scom_addresses.H> +#include <p9a_misc_scom_addresses_fld.H> +#include <chips/ocmb/explorer/procedures/hwp/memory/exp_inband.H> + +//----------------------------------------------------------------------------------- +// Function definitions +//----------------------------------------------------------------------------------- + +fapi2::ReturnCode p9a_omi_setup_bars( + const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target) +{ + FAPI_DBG("Start"); + std::vector<uint64_t> l_base_addr_nm0; + std::vector<uint64_t> l_base_addr_nm1; + std::vector<uint64_t> l_base_addr_m; + fapi2::buffer<uint64_t> l_mmio_bar; + fapi2::buffer<uint64_t> l_cfg_bar; + uint64_t l_base_addr_mmio; + uint8_t l_pos; + uint8_t l_mcc_pos; + uint64_t l_ext_mask; + + // determine base address of chip MMIO range + FAPI_TRY(p9_fbc_utils_get_chip_base_address(i_target, + EFF_FBC_GRP_CHIP_IDS, + l_base_addr_nm0, + l_base_addr_nm1, + l_base_addr_m, + l_base_addr_mmio), + "Error from p9_fbc_utils_get_chip_base_address"); + + FAPI_TRY(p9a_get_ext_mask(l_ext_mask)); + + FAPI_DBG("l_ext_mask: %x", l_ext_mask); + FAPI_DBG("l_base_addr_mmio: 0x%llx", l_base_addr_mmio); + + for (auto l_mcc : i_target.getChildren<fapi2::TARGET_TYPE_MCC>()) + { + fapi2::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET_Type l_bar_offset; + uint64_t l_omi_inband_addr = 0; + fapi2::buffer<uint64_t> l_scom_data; + + auto l_omi_targets = l_mcc.getChildren<fapi2::TARGET_TYPE_OMI>(); + + if (l_omi_targets.size() > 0) + { + + // Set the sizes for the MMIO/Config bars + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mcc, l_pos), + "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X", + (uint64_t)fapi2::current_err); + + l_scom_data.flush<0>(); + // 2GB cfg and MMIO + l_scom_data.insertFromRight<P9A_MI_MCFGP0_MCFGPR0_CONFIGURATION_GROUP_SIZE, + P9A_MI_MCFGP0_MCFGPR0_CONFIGURATION_GROUP_SIZE_LEN>(mss::exp::ib::EXPLR_IB_BAR_SIZE); + l_scom_data.insertFromRight<P9A_MI_MCFGP0_MCFGPR0_MMIO_GROUP_SIZE, + P9A_MI_MCFGP0_MCFGPR0_MMIO_GROUP_SIZE_LEN>(mss::exp::ib::EXPLR_IB_BAR_SIZE); + + // Write to reg + if (l_pos % 2 == 0) + { + FAPI_INF("Write MCFGP0 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP0, l_scom_data); + FAPI_TRY(fapi2::putScom(l_mcc.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP0, l_scom_data), + "Error writing to MCS_MCFGP reg"); + } + else + { + FAPI_INF("Write MCFGP1 reg 0x%.16llX, Value 0x%.16llX", + P9A_MI_MCFGP1, l_scom_data); + FAPI_TRY(fapi2::putScom(l_mcc.getParent<fapi2::TARGET_TYPE_MI>(), P9A_MI_MCFGP1, l_scom_data), + "Error writing to MCS_MCFGP reg"); + } + + for (auto l_omi : l_omi_targets) + { + // retrieve OMI pos + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, + l_omi, + l_pos), + "Error from FAPI_ATTR_GET (ATTR_CHIP_UNIT_POS)"); + + // retrieve inband BAR offset + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET, + l_omi, + l_bar_offset), + "Error from FAPI_ATTR_GET (ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET)"); + + FAPI_DBG("l_bar_offset: 0x%llx", l_bar_offset); + + // If this is channel B, then the B bit must be set. If it is not, then the B bit must not be set. + FAPI_ASSERT( ((l_pos % 2) == 1) == // Is this B channel? + ((l_bar_offset & mss::exp::ib::EXPLR_IB_BAR_B_BIT) == mss::exp::ib::EXPLR_IB_BAR_B_BIT), + + fapi2::PROC_OMI_SETUP_BARS_INVALID_BAR() + .set_TARGET(l_omi) + .set_BAR_VALUE(l_bar_offset), + "B Channel requires BAR size bit set"); + + FAPI_ASSERT(((l_bar_offset & mss::exp::ib::EXPLR_IB_BAR_MASK_ZERO) == 0), + fapi2::PROC_OMI_SETUP_BARS_INVALID_BAR() + .set_TARGET(l_omi) + .set_BAR_VALUE(l_bar_offset), + "Bar size not honored"); + + FAPI_ASSERT(((l_bar_offset & mss::exp::ib::EXPLR_IB_MMIO_OFFSET) == 0), + fapi2::PROC_OMI_SETUP_BARS_INVALID_BAR() + .set_TARGET(l_omi) + .set_BAR_VALUE(l_bar_offset), + "MMIO bit must not be set"); + + l_omi_inband_addr = l_bar_offset; + } + + FAPI_DBG("l_omi_inband_addr: 0x%llx", l_omi_inband_addr); + + // Force A Bar value + l_omi_inband_addr = l_omi_inband_addr & (~mss::exp::ib::EXPLR_IB_BAR_B_BIT); + + FAPI_DBG("l_omi_inband_addr: 0x%llx", l_omi_inband_addr); + + // Add MMIO address for the chip + l_omi_inband_addr = l_omi_inband_addr | l_base_addr_mmio; + + FAPI_DBG("l_omi_inband_addr: 0x%llx", l_omi_inband_addr); + + // Get cfg bar value + fapi2::buffer<uint64_t> l_omi_inband_buf = l_omi_inband_addr; + FAPI_DBG("l_omi_inband_buf: 0x%llx", l_omi_inband_buf); + FAPI_TRY(extendBarAddress(l_ext_mask, l_omi_inband_buf, l_cfg_bar)); + + FAPI_DBG("l_cfg_bar: 0x%llx", l_cfg_bar); + + // Get MMIO bar value + l_omi_inband_addr = l_omi_inband_addr | mss::exp::ib::EXPLR_IB_MMIO_OFFSET; + l_omi_inband_buf = l_omi_inband_addr; + FAPI_DBG("l_omi_inband_buf: 0x%llx", l_omi_inband_buf); + FAPI_TRY(extendBarAddress(l_ext_mask, l_omi_inband_buf, l_mmio_bar)); + + FAPI_DBG("l_mmio_bar: 0x%llx", l_mmio_bar); + + // Write the channel cfg reg + l_scom_data.flush<0>(); + l_scom_data.setBit<P9A_MI_MCFGPR0_CONFIGURATION_VALID>(); //Enable CFG + l_scom_data.setBit<P9A_MI_MCFGPR0_MMIO_VALID>(); //Enable MMIO + l_scom_data.insert<P9A_MI_MCFGPR0_CONFIGURATION_GROUP_BASE_ADDRESS, + P9A_MI_MCFGPR0_CONFIGURATION_GROUP_BASE_ADDRESS_LEN>(l_cfg_bar); + l_scom_data.insert<P9A_MI_MCFGPR0_MMIO_GROUP_BASE_ADDRESS, + P9A_MI_MCFGPR0_MMIO_GROUP_BASE_ADDRESS_LEN>(l_mmio_bar); + + // get MI target to configure MCFGPR + fapi2::Target<fapi2::TARGET_TYPE_MI> l_mi = + l_mcc.getParent<fapi2::TARGET_TYPE_MI>(); + // retrieve DMI pos + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, + l_mcc, + l_mcc_pos), + "Error from FAPI_ATTR_GET (ATTR_CHIP_UNIT_POS)"); + + // configure inband channel 0 MCFGPR0 + if(l_mcc_pos % 2 == 0) + { + FAPI_TRY(putScom(l_mi, P9A_MI_MCFGPR0, l_scom_data)); + } + // configure inband channel 1 MCFGPR1 + else + { + FAPI_TRY(putScom(l_mi, P9A_MI_MCFGPR1, l_scom_data)); + } + } + } + +fapi_try_exit: + FAPI_DBG("End"); + return fapi2::current_err; +} diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.H b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.H new file mode 100644 index 000000000..b313bf2d0 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.H @@ -0,0 +1,74 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.H $ */ +/* */ +/* OpenPOWER HostBoot Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2017,2019 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9a_omi_setup_bars.H +/// @brief configure Axone inband address +/// +/// Set the inband base address in the MC +/// driven by attributes representing system memory map. +/// + +// *HWP HWP Owner: Ben Gass bgass@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 3 +// *HWP Consumed by: HB + +#ifndef _p9a_omi_setup_bars_H_ +#define _p9a_omi_setup_bars_H_ + + +//----------------------------------------------------------------------------------- +// Includes +//----------------------------------------------------------------------------------- +#include <fapi2.H> + +//----------------------------------------------------------------------------------- +// Structure definitions +//----------------------------------------------------------------------------------- + +// function pointer typedef definition for HWP call support +typedef fapi2::ReturnCode (*p9a_omi_setup_bars_FP_t) ( + const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>&); + +//----------------------------------------------------------------------------------- +// Function prototype +//----------------------------------------------------------------------------------- + +extern "C" { + +/// +/// @brief configure Cumulus inband address +/// +/// @param[in] i_target => Processor chip target +/// +/// @return FAPI_RC_SUCCESS if the setup completes successfully, else error +// + fapi2::ReturnCode p9a_omi_setup_bars( + const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target); + +} //extern"C" + +#endif //_p9a_omi_setup_bars_H_ diff --git a/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.mk b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.mk new file mode 100644 index 000000000..d24e4e5f0 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.mk @@ -0,0 +1,29 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/import/chips/p9/procedures/hwp/nest/p9a_omi_setup_bars.mk $ +# +# OpenPOWER HostBoot Project +# +# Contributors Listed Below - COPYRIGHT 2017,2019 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +PROCEDURE=p9a_omi_setup_bars +OBJS+=p9_fbc_utils.o +OBJS+=p9a_addr_ext.o +$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)) +$(call BUILD_PROCEDURE) |
