p9_throttle_sync -- initial version for Cumulus

1) Mostly copied from Nimbus settings.
2) Add findNumDimms() for dimm numbers.
3) Add MCMODE0 bit 27/28 programming for Nimbus DD2
   and Cumulus.
4) No HW397255 workaround for Nimbus DD2 and Cumulus
5) Generalize throttleSync(), progMaster()
   and progMCMODE0().

Change-Id: I74d96f4113cb2a33eaba6e1966b17aac693c9c4e
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39695
Reviewed-by: Joseph J. McGill <jmcgill@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39712
Reviewed-by: Hostboot Team <hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 206 insertions, 67 deletions

diff --git a/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H b/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H
index 69deff6da..ef6af5669 100644
--- a/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H
+++ b/src/import/chips/p9/common/include/p9_mc_scom_addresses_fld_fixes.H
@@ -68,6 +68,10 @@ REG64_FLD( MCS_MCMODE0_GROUP_INTERLEAVE_GRANULARITY                , 52  , SH_UN
            0 );
 REG64_FLD( MCS_MCMODE0_GROUP_INTERLEAVE_GRANULARITY_LEN            , 4   , SH_UNT_MCS      , SH_ACS_SCOM_RW  ,
            0 );
+REG64_FLD( MCS_MCMODE0_DISABLE_MC_SYNC                             , 27  , SH_UNT_MCS      , SH_ACS_SCOM_RW  ,
+           0 );
+REG64_FLD( MCS_MCMODE0_DISABLE_MC_PAIR_SYNC                        , 28  , SH_UNT_MCS      , SH_ACS_SCOM_RW  ,
+           0 );
 
 
 
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C b/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C
index 28882ad36..e2715b72f 100644
--- a/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_throttle_sync.C
@@ -44,34 +44,178 @@
 #include <p9_throttle_sync.H>
 #include <fapi2.H>
 #include <generic/memory/lib/utils/find.H>
+#include <p9_perv_scom_addresses.H>
 
 ///----------------------------------------------------------------------------
 /// Constant definitions
 ///----------------------------------------------------------------------------
 const uint8_t SUPER_SYNC_BIT   = 14;
 const uint8_t MAX_MC_SIDES_PER_PROC = 2; // MC01, MC23
+const uint8_t MAX_MC_PER_PROC = 4; // MC0, MC1, MC2, MC3
+const uint8_t MAX_MC_PER_SIDE = 2; // MC0, MC1 or MC2, MC3
 
-// Structure that holds the potential master MCS for a MC side (MC01/MC23)
+// Structure that holds the potential master MCS or MI for a MC side (MC01/MC23)
+template <fapi2::TargetType T>
 struct mcSideInfo_t
 {
-    bool masterMcsFound = false;
-    // Master MCS for this MC side
-    fapi2::Target<fapi2::TARGET_TYPE_MCS> masterMcs;
+    bool masterMcFound = false;
+
+    // Master MC for this MC side
+    fapi2::Target<T> masterMc;
 };
 
+///
+/// @brief Find out how many DIMMS are connected to a MCS or MI target
+///
+/// @tparam T   template parameter, passed in targets.
+/// @param[in]  i_miTarget   The MI target to be programmed as master
+///
+/// @return uint8_t number of DIMMs
+///
+template< fapi2::TargetType T>
+uint8_t findNumDimms(const fapi2::Target<T>& i_mcTarget);
+
+template<>
+uint8_t findNumDimms(const fapi2::Target<fapi2::TARGET_TYPE_MI>& i_miTarget)
+{
+    FAPI_DBG("Entering findNumDimms");
+    auto l_dmiChiplets = i_miTarget.getChildren<fapi2::TARGET_TYPE_DMI>();
+
+    uint8_t l_num_dimms = 0;
+
+    for (auto l_dmi : l_dmiChiplets)
+    {
+        auto l_memBufs = l_dmi.getChildren<fapi2::TARGET_TYPE_MEMBUF_CHIP>();
+
+        if (l_memBufs.size() > 0)
+        {
+            l_num_dimms++;
+        }
+    }
+
+    FAPI_DBG("Exiting findNumDimms");
+    return l_num_dimms;
+}
+
+template<>
+uint8_t findNumDimms(const fapi2::Target<fapi2::TARGET_TYPE_MCS>& i_mcsTarget)
+{
+
+    FAPI_DBG("Entering findNumDimms");
+    std::vector< fapi2::Target<fapi2::TARGET_TYPE_DIMM> > l_dimms =
+        mss::find_targets<fapi2::TARGET_TYPE_DIMM>(i_mcsTarget);
+
+    FAPI_DBG("Exiting findNumDimms");
+    return l_dimms.size();
+}
+
+///
+/// @brief Program MCMODE0 based on the functional targets
+///
+/// @tparam T template parameter, passed in targets.
+/// @param[in]  i_mcTarget   The MC target to be programmed
+/// @param[in]  i_mcTargets  Other MC targets.
+///
+/// @return FAPI2_RC_SUCCESS if success, else error code.
+///
+template< fapi2::TargetType T>
+fapi2::ReturnCode progMCMODE0(
+    fapi2::Target<T>& i_mcTarget,
+    const std::vector< fapi2::Target<T> >& i_mcTargets)
+{
+    FAPI_DBG("Entering progMCMODE0");
+    // --------------------------------------------------------------
+    // Setup MCMODE0 for disabling MC SYNC to other-side and same-side
+    // partner unit.
+    // BIT27: set if other-side MC is non-functional, 0<->2, 1<->3
+    // BIT28: set if same-side MC is non-functional, 0<->1, 2<->3
+    // --------------------------------------------------------------
+    fapi2::buffer<uint64_t> l_scomData(0);
+    fapi2::buffer<uint64_t> l_scomMask(0);
+    bool l_other_side_functional = false;
+    bool l_same_side_functional = false;
+    uint8_t l_current_pos = 0;
+    uint8_t l_other_side_pos = 0;
+    uint8_t l_same_side_pos = 0;
+
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_mcTarget,
+                           l_current_pos),
+             "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X",
+             (uint64_t)fapi2::current_err);
+
+    // Calculate the peer MC in the other side and in the same side.
+    l_other_side_pos = (l_current_pos + MAX_MC_PER_SIDE) % MAX_MC_PER_PROC;
+    l_same_side_pos = ((l_current_pos / MAX_MC_SIDES_PER_PROC) * MAX_MC_PER_SIDE)
+                      + ((l_current_pos % MAX_MC_PER_SIDE) + 1) % MAX_MC_PER_SIDE;
+
+    FAPI_DBG("Current pos: %i, other side pos: %i, same side pos: %i",
+             l_current_pos, l_other_side_pos, l_same_side_pos);
+
+    // Determine side functionality
+    for (auto l_mc : i_mcTargets)
+    {
+
+        uint8_t l_tmp_pos = 0;
+        FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc,
+                               l_tmp_pos),
+                 "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X",
+                 (uint64_t)fapi2::current_err);
+
+        // The other side
+        if (l_tmp_pos == l_other_side_pos)
+        {
+            l_other_side_functional = true;
+        }
+
+        // the same side
+        if (l_tmp_pos == l_same_side_pos)
+        {
+            l_same_side_functional = true;
+        }
+    }
+
+    l_scomData.flush<0>();
+    l_scomMask.flush<0>();
+
+    if (!l_other_side_functional)
+    {
+        l_scomData.setBit<MCS_MCMODE0_DISABLE_MC_SYNC>();
+        l_scomMask.setBit<MCS_MCMODE0_DISABLE_MC_SYNC>();
+    }
+
+    if (!l_same_side_functional)
+    {
+        l_scomData.setBit<MCS_MCMODE0_DISABLE_MC_PAIR_SYNC>();
+        l_scomMask.setBit<MCS_MCMODE0_DISABLE_MC_PAIR_SYNC>();
+    }
+
+    FAPI_INF("Writing MCS_MCMODE0 reg 0x%.16llX: Mask 0x%.16llX , Data 0x%.16llX",
+             MCS_MCMODE0, l_scomMask, l_scomData);
+
+    FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCMODE0,
+                                     l_scomData, l_scomMask),
+             "putScomUnderMask() returns an error (Mode0), MCS_MCMODE0 reg 0x%.16llX",
+             MCS_MCMODE0);
+
+fapi_try_exit:
+    FAPI_DBG("Exiting progMCMODE0");
+    return fapi2::current_err;
+}
 
 ///
 /// @brief Programming master MCS
 ///        Writes MCS_MCSYNC reg to set the input MCS as the master.
 ///
+/// @tparam T template parameter, passed in targets.
 /// @param[in]  i_mcsTarget  The MCS target to be programmed as master.
 ///
 /// @return FAPI2_RC_SUCCESS if success, else error code.
 ///
-fapi2::ReturnCode progMasterMcs(
-    const fapi2::Target<fapi2::TARGET_TYPE_MCS>& i_mcsTarget)
+template< fapi2::TargetType T>
+fapi2::ReturnCode progMaster(
+    const fapi2::Target<T>& i_mcTarget)
 {
-    FAPI_DBG("Entering");
+    FAPI_DBG("Entering progMaster");
     fapi2::ReturnCode l_rc;
     fapi2::buffer<uint64_t> l_scomData(0);
     fapi2::buffer<uint64_t> l_scomMask(0);
@@ -96,13 +240,13 @@ fapi2::ReturnCode progMasterMcs(
 
     l_scomMask.flush<0>().setBit<MCS_MCSYNC_SYNC_GO_CH0>();
     l_scomData.flush<0>();
-    FAPI_TRY(fapi2::putScomUnderMask(i_mcsTarget, MCS_MCSYNC,
+    FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCSYNC,
                                      l_scomData, l_scomMask),
              "putScomUnderMask() returns an error (Sync reset), Addr 0x%.16llX",
              MCS_MCSYNC);
 
     // --------------------------------------------------------------
-    // 2. Setup MC Sync Command Register data for master MCS
+    // 2. Setup MC Sync Command Register data for master MCS or MI
     // --------------------------------------------------------------
     // Clear buffers
     l_scomData.flush<0>();
@@ -117,6 +261,8 @@ fapi2::ReturnCode progMasterMcs(
 
     // Setup MCSYNC_SYNC_TYPE
     // Set all sync types except Super Sync
+    // SUPER_SYNC_BIT == bit 14, supersync for Nimbus, reserved for cumulus.
+    // Clear it in both cases.
     l_scomData.setBit<MCS_MCSYNC_SYNC_TYPE,
                       MCS_MCSYNC_SYNC_TYPE_LEN>().clearBit(SUPER_SYNC_BIT);
     l_scomMask.setBit<MCS_MCSYNC_SYNC_TYPE,
@@ -127,13 +273,13 @@ fapi2::ReturnCode progMasterMcs(
     l_scomData.setBit<MCS_MCSYNC_SYNC_GO_CH0>();
 
     // --------------------------------------------------------------
-    // 3. Write to MC Sync Command Register of master MCS
+    // 3. Write to MC Sync Command Register of master MCS or MI
     // --------------------------------------------------------------
     // Write to MCSYNC reg
     FAPI_INF("Writing MCS_MCSYNC reg 0x%.16llX: Mask 0x%.16llX , Data 0x%.16llX",
              MCS_MCSYNC, l_scomMask, l_scomData);
 
-    FAPI_TRY(fapi2::putScomUnderMask(i_mcsTarget, MCS_MCSYNC,
+    FAPI_TRY(fapi2::putScomUnderMask(i_mcTarget, MCS_MCSYNC,
                                      l_scomData, l_scomMask),
              "putScomUnderMask() returns an error (Sync), MCS_MCSYNC reg 0x%.16llX",
              MCS_MCSYNC);
@@ -141,31 +287,27 @@ fapi2::ReturnCode progMasterMcs(
     // Note: No need to read Sync replay count and retry in P9.
 
 fapi_try_exit:
-    FAPI_DBG("Exiting");
+    FAPI_DBG("Exiting progMaster");
     return fapi2::current_err;
 }
 
-
 ///
 /// @brief Perform throttle sync on the Memory Controllers
 ///
 /// @tparam T template parameter, passed in targets.
-/// @param[in]  i_mcTargets     Vector of reference of MC targets (MCS or MI)
+/// @param[in]  i_mcTargets        Vector of reference of MC targets (MCS or MI)
+/// @param[in]  i_HW397255_enabled Workaround for HW397255
 ///
 /// @return FAPI2_RC_SUCCESS if success, else error code.
 ///
 template< fapi2::TargetType T>
 fapi2::ReturnCode throttleSync(
-    const std::vector< fapi2::Target<T> >& i_mcTargets);
-
-/// TARGET_TYPE_MCS
-template<>
-fapi2::ReturnCode throttleSync(
-    const std::vector< fapi2::Target<fapi2::TARGET_TYPE_MCS> >& i_mcTargets)
+    const std::vector< fapi2::Target<T> >& i_mcTargets,
+    fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type i_HW397255_enabled)
 {
     FAPI_DBG("Entering");
     fapi2::ReturnCode l_rc;
-    mcSideInfo_t l_mcSide[MAX_MC_SIDES_PER_PROC];
+    mcSideInfo_t<T> l_mcSide[MAX_MC_SIDES_PER_PROC];
     uint8_t l_sideNum = 0;
     uint8_t l_pos = 0;
     uint8_t l_numMasterProgrammed = 0;
@@ -173,25 +315,24 @@ fapi2::ReturnCode throttleSync(
     // Initialization
     for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++)
     {
-        l_mcSide[l_sideNum].masterMcsFound = false;
+        l_mcSide[l_sideNum].masterMcFound = false;
     }
 
     // ---------------------------------------------------------------------
-    // 1. Pick the first MCS with DIMMS as potential master
+    // 1. Pick the first MCS/MI with DIMMS as potential master
     //    for both MC sides (MC01/MC23)
     // ---------------------------------------------------------------------
-    for (auto l_mcs : i_mcTargets)
+    for (auto l_mc : i_mcTargets)
     {
-        std::vector< fapi2::Target<fapi2::TARGET_TYPE_DIMM> > l_dimms =
-            mss::find_targets<fapi2::TARGET_TYPE_DIMM>(l_mcs);
+        uint8_t l_num_dimms = findNumDimms(l_mc);
 
-        if (l_dimms.size() > 0)
+        if (l_num_dimms > 0)
         {
-            // This MCS has DIMMs attached, find out which MC side it
+            // This MCS or MI has DIMMs attached, find out which MC side it
             // belongs to:
             //    l_sideNum = 0 --> MC01
             //                1 --> MC23
-            FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mcs,
+            FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc,
                                    l_pos),
                      "Error getting ATTR_CHIP_UNIT_POS, l_rc 0x%.8X",
                      (uint64_t)fapi2::current_err);
@@ -199,52 +340,51 @@ fapi2::ReturnCode throttleSync(
 
             FAPI_INF("MCS %u has DIMMs", l_pos);
 
-            // If there's no master MCS marked for this side yet, mark
+            // If there's no master MCS or MI marked for this side yet, mark
             // this MCS as master
-            if (l_mcSide[l_sideNum].masterMcsFound == false)
+            if (l_mcSide[l_sideNum].masterMcFound == false)
             {
                 FAPI_INF("Mark MCS %u as master for MC side %u",
                          l_pos, l_sideNum);
-                l_mcSide[l_sideNum].masterMcsFound = true;
-                l_mcSide[l_sideNum].masterMcs = l_mcs;
+                l_mcSide[l_sideNum].masterMcFound = true;
+                l_mcSide[l_sideNum].masterMc = l_mc;
             }
         }
+
+        // Program the MCMODE0 if HW397255 is not enabled which means we
+        // should have a chip with Nimbus DD2+ or Cumulus.
+        if (i_HW397255_enabled == 0)
+        {
+            progMCMODE0(l_mc, i_mcTargets);
+        }
     }
 
     // --------------------------------------------------------------
-    // 2. Program the master MCS
+    // 2. Program the master MCS or MI
     // --------------------------------------------------------------
     for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++)
     {
-        // If there is a potential master MCS found for this side
-        if (l_mcSide[l_sideNum].masterMcsFound == true)
+        // If there is a potential master MCS or MI found for this side
+        if (l_mcSide[l_sideNum].masterMcFound == true)
         {
-            // No master MCS programmed for either side yet,
-            // go ahead and program this MCS as master.
+            // No master MCS or MI programmed for either side yet,
+            // go ahead and program this MCS or MI as master.
             if (l_numMasterProgrammed == 0)
             {
-                FAPI_TRY(progMasterMcs(l_mcSide[l_sideNum].masterMcs),
-                         "programMasterMcs() returns error"
+                FAPI_TRY(progMaster(l_mcSide[l_sideNum].masterMc),
+                         "programMaster() returns error"
                          "NumMasterProgrammed %d, l_rc 0x%.8X",
                          l_numMasterProgrammed, (uint64_t)fapi2::current_err);
                 l_numMasterProgrammed++;
             }
             else
             {
-                // A master MCS is already programmed on MC side 0 (MC01).
                 // HW397255 requires to also program a master MCS on MC23 if
                 // it has DIMMs.
-                fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_chip =
-                    l_mcSide[l_sideNum].masterMcs.getParent<fapi2::TARGET_TYPE_PROC_CHIP>();
-                fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type l_HW397255_enabled;
-                FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_HW397255,
-                                       l_chip, l_HW397255_enabled),
-                         "Error getting the ATTR_CHIP_EC_FEATURE_HW397255");
-
-                if (l_HW397255_enabled)
+                if (i_HW397255_enabled == 1)
                 {
-                    FAPI_TRY(progMasterMcs(l_mcSide[l_sideNum].masterMcs),
-                             "programMasterMcs() returns error"
+                    FAPI_TRY(progMaster(l_mcSide[l_sideNum].masterMc),
+                             "programMaster() returns error"
                              "NumMasterProgrammed %d, l_rc 0x%.8X",
                              l_numMasterProgrammed, (uint64_t)fapi2::current_err);
                 }
@@ -257,20 +397,6 @@ fapi_try_exit:
     return fapi2::current_err;
 }
 
-/// TARGET_TYPE_MI
-template<>
-fapi2::ReturnCode throttleSync(
-    const std::vector< fapi2::Target<fapi2::TARGET_TYPE_MI> >& i_mcTargets)
-{
-    FAPI_DBG("Entering");
-    fapi2::ReturnCode l_rc;
-
-    // Note: Add code for Cumulus
-
-    FAPI_DBG("Exiting");
-    return fapi2::current_err;
-}
-
 extern "C"
 {
 
@@ -291,10 +417,19 @@ extern "C"
         auto l_mcsChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MCS>();
         auto l_miChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MI>();
 
+        // HW397255 requires to also program a master MCS on MC23 if
+        // it has DIMMs.
+        // This should only be enabled for Nimbus DD1, disabled for Nimbus DD2
+        // and Cumulus.
+        fapi2::ATTR_CHIP_EC_FEATURE_HW397255_Type l_HW397255_enabled;
+        FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_EC_FEATURE_HW397255,
+                               i_target, l_HW397255_enabled),
+                 "Error getting the ATTR_CHIP_EC_FEATURE_HW397255");
+
         // Get the functional MCS on this proc
         if (l_mcsChiplets.size() > 0)
         {
-            FAPI_TRY(throttleSync(l_mcsChiplets),
+            FAPI_TRY(throttleSync(l_mcsChiplets, l_HW397255_enabled),
                      "throttleSync() returns error l_rc 0x%.8X",
                      (uint64_t)fapi2::current_err);
         }
@@ -302,7 +437,7 @@ extern "C"
         // Cumulus
         if (l_miChiplets.size() > 0)
         {
-            FAPI_TRY(throttleSync(l_miChiplets),
+            FAPI_TRY(throttleSync(l_miChiplets, l_HW397255_enabled),
                      "throttleSync() returns error l_rc 0x%.8X",
                      (uint64_t)fapi2::current_err);
         }