Disable BAR Temporarily to prevent STR exit during nvdimm handover

Change-Id: Ib7e95d8aa0b77570fec9c76211e78d790f540b06
CQ:SW468134
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79219
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: STEPHEN GLANCY <sglancy@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79375
Reviewed-by: RAJA DAS <rajadas2@in.ibm.com>

2 files changed, 85 insertions, 27 deletions

diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.C b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.C
index 9621e20b..c5914c3c 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.C
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.C
@@ -103,6 +103,15 @@ constexpr const uint64_t FARB6Q_REG[] =
     MCA_7_MBA_FARB6Q,
 };
 
+// MCFGP for each MCS
+constexpr const uint64_t MCFGP[] =
+{
+    MCS_0_MCFGP,
+    MCS_1_MCFGP,
+    MCS_2_MCFGP,
+    MCS_3_MCFGP,
+};
+
 // MCB_CNTLQ
 constexpr const uint64_t MCB_CNTLQ_REG[] =
 {
@@ -117,19 +126,19 @@ constexpr uint8_t PORTS_PER_MODULE = 8;
 /// This is the same thing as mss::nvdimm::self_refresh_entry() but rewritten
 /// to support SBE
 /// @param[in] i_target - PROC_CHIP target
-/// @param[in] l_mca_pos - The MCA position relative to the PROC
+/// @param[in] i_mca_pos - The MCA position relative to the PROC
 /// @return FAPI2_RC_SUCCESS iff setup was successful
 ///
 fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
-                                      const uint8_t l_mca_pos)
+                                      const uint8_t i_mca_pos)
 {
-    FAPI_ASSERT((l_mca_pos < PORTS_PER_MODULE),
+    FAPI_ASSERT((i_mca_pos < PORTS_PER_MODULE),
                 fapi2::MSS_SRE_MCA_OUT_OF_RANGE().
                 set_PROC_TARGET(i_target).
-                set_MCA_POS(l_mca_pos),
-                "Invalid port number %u provided", l_mca_pos);
+                set_MCA_POS(i_mca_pos),
+                "Invalid port number %u provided", i_mca_pos);
 
-    FAPI_DBG("Entering STR on port %u.", l_mca_pos);
+    FAPI_DBG("Entering STR on port %u.", i_mca_pos);
 
     {
         fapi2::buffer<uint64_t> l_mbarpc0_data;
@@ -142,7 +151,7 @@ fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PRO
         constexpr uint64_t STOP = 1;
         constexpr uint64_t PORTS_PER_MCBIST = 4;
         constexpr uint64_t TIME_0 = 0;
-        const uint8_t l_mcbist = l_mca_pos < PORTS_PER_MCBIST ? 0 : 1;
+        const uint8_t l_mcbist = i_mca_pos < PORTS_PER_MCBIST ? 0 : 1;
 
         // Variables for polling. Poll up to a second to make sure the port has entered STR
         // Why a second? No idea. STR is controlled by the sequencer via power control.
@@ -160,21 +169,21 @@ fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PRO
 
         // Step 1 - In MBARPC0Q, disable power domain control, set domain to MAXALL_MIN0,
         //          and disable minimum domain reduction (allow immediate entry of STR)
-        FAPI_TRY(fapi2::getScom(i_target, MBARPC0Q_REG[l_mca_pos], l_mbarpc0_data));
+        FAPI_TRY(fapi2::getScom(i_target, MBARPC0Q_REG[i_mca_pos], l_mbarpc0_data));
         l_mbarpc0_data.writeBit<MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS_ENABLE>(DISABLE);
         l_mbarpc0_data.insertFromRight<MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS, MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS_LEN>(MAXALL_MIN0);
         l_mbarpc0_data.writeBit<MCA_MBARPC0Q_CFG_MIN_DOMAIN_REDUCTION_ENABLE>(DISABLE);
-        FAPI_TRY(fapi2::putScom(i_target, MBARPC0Q_REG[l_mca_pos], l_mbarpc0_data));
+        FAPI_TRY(fapi2::putScom(i_target, MBARPC0Q_REG[i_mca_pos], l_mbarpc0_data));
 
         // Step 2 - In MBASTR0Q, enable STR entry
-        FAPI_TRY(fapi2::getScom(i_target, MBASTR0Q_REG[l_mca_pos], l_mbastr0_data));
+        FAPI_TRY(fapi2::getScom(i_target, MBASTR0Q_REG[i_mca_pos], l_mbastr0_data));
         l_mbastr0_data.writeBit<MCA_MBASTR0Q_CFG_STR_ENABLE>(ENABLE);
         l_mbastr0_data.insertFromRight<MCA_MBASTR0Q_CFG_ENTER_STR_TIME, MCA_MBASTR0Q_CFG_ENTER_STR_TIME_LEN>(TIME_0);
-        FAPI_TRY(fapi2::putScom(i_target, MBASTR0Q_REG[l_mca_pos], l_mbastr0_data));
+        FAPI_TRY(fapi2::putScom(i_target, MBASTR0Q_REG[i_mca_pos], l_mbastr0_data));
 
         // Step 3 - In MBARPC0Q, enable power domain control.
         l_mbarpc0_data.writeBit<MCA_MBARPC0Q_CFG_MIN_MAX_DOMAINS_ENABLE>(ENABLE);
-        FAPI_TRY(fapi2::putScom(i_target, MBARPC0Q_REG[l_mca_pos], l_mbarpc0_data));
+        FAPI_TRY(fapi2::putScom(i_target, MBARPC0Q_REG[i_mca_pos], l_mbarpc0_data));
 
 #ifndef __PPE__
         FAPI_TRY( FAPI_ATTR_GET(fapi2::ATTR_IS_SIMULATION, fapi2::Target<fapi2::TARGET_TYPE_SYSTEM>(), l_sim) );
@@ -186,7 +195,7 @@ fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PRO
             for (uint8_t i = 0; i < POLL_ITERATION; i++)
             {
                 FAPI_TRY(fapi2::delay(DELAY_2MS_IN_NS, 0), "Error returned from fapi2::delay call");
-                FAPI_TRY(fapi2::getScom(i_target, FARB6Q_REG[l_mca_pos], l_mcafarb6q_data));
+                FAPI_TRY(fapi2::getScom(i_target, FARB6Q_REG[i_mca_pos], l_mcafarb6q_data));
 
                 if (l_mcafarb6q_data.getBit<MCA_MBA_FARB6Q_CFG_STR_STATE>())
                 {
@@ -198,10 +207,10 @@ fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PRO
             FAPI_ASSERT(l_in_str,
                         fapi2::MSS_STR_NOT_ENTERED().
                         set_PROC_TARGET(i_target).
-                        set_MCA_POS(l_mca_pos).
+                        set_MCA_POS(i_mca_pos).
                         set_MCA_FARB6Q(l_mcafarb6q_data).
                         set_STR_STATE(l_in_str),
-                        "STR not entered on port %u", l_mca_pos);
+                        "STR not entered on port %u", i_mca_pos);
         }
     }
 
@@ -213,26 +222,54 @@ fapi_try_exit:
 /// @brief Helper for trigger_csave. This subroutine assert RESET_n to trigger
 /// the backup on nvdimm
 /// @param[in] i_target - PROC_CHIP targetdefine
-/// @param[in] l_mca_pos - The MCA position relative to the PROC
+/// @param[in] i_mca_pos - The MCA position relative to the PROC
 /// @return FAPI2_RC_SUCCESS iff setup was successful
 ///
-fapi2::ReturnCode assert_resetn( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, const uint8_t l_mca_pos)
+fapi2::ReturnCode assert_resetn( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, const uint8_t i_mca_pos)
 {
-    FAPI_ASSERT((l_mca_pos < PORTS_PER_MODULE),
+    FAPI_ASSERT((i_mca_pos < PORTS_PER_MODULE),
                 fapi2::MSS_RESETN_MCA_OUT_OF_RANGE().
                 set_PROC_TARGET(i_target).
-                set_MCA_POS(l_mca_pos),
-                "Invalid port number %u provided", l_mca_pos);
+                set_MCA_POS(i_mca_pos),
+                "Invalid port number %u provided", i_mca_pos);
 
-    FAPI_DBG("Asserting RESETn on port %d.", l_mca_pos);
+    FAPI_DBG("Asserting RESETn on port %d.", i_mca_pos);
 
     {
         fapi2::buffer<uint64_t> l_farb5q_data;
         constexpr uint64_t LOW = 0;
 
-        FAPI_TRY(fapi2::getScom(i_target, FARB5Q_REG[l_mca_pos], l_farb5q_data));
+        FAPI_TRY(fapi2::getScom(i_target, FARB5Q_REG[i_mca_pos], l_farb5q_data));
         l_farb5q_data.writeBit<MCA_MBA_FARB5Q_CFG_DDR_RESETN>(LOW);
-        FAPI_TRY(fapi2::putScom(i_target, FARB5Q_REG[l_mca_pos], l_farb5q_data));
+        FAPI_TRY(fapi2::putScom(i_target, FARB5Q_REG[i_mca_pos], l_farb5q_data));
+    }
+
+fapi_try_exit:
+    return fapi2::current_err;
+}
+
+///
+/// @brief change the BAR valid state on MCS with NVDIMM installed
+/// @param[in] i_target - PROC_CHIP target
+/// @param[in] i_port_bitmap - port bitmap that indicates port with nvdimm
+/// @param[in] i_state - BAR state to change to
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+fapi2::ReturnCode change_bar_valid_state( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+        const fapi2::buffer<uint8_t> i_port_bitmap,
+        const uint64_t i_state)
+{
+    for (uint8_t l_mca_pos = 0; l_mca_pos < PORTS_PER_MODULE; l_mca_pos++)
+    {
+        if (i_port_bitmap.getBit(l_mca_pos) == fapi2::ENUM_ATTR_SBE_NVDIMM_IN_PORT_YES)
+        {
+            const uint8_t l_mcs = l_mca_pos / 2;
+            fapi2::buffer<uint64_t> l_data;
+
+            FAPI_TRY(fapi2::getScom(i_target, MCFGP[l_mcs], l_data));
+            l_data.writeBit<MCS_MCFGP_VALID>(i_state);
+            FAPI_TRY(fapi2::putScom(i_target, MCFGP[l_mcs], l_data));
+        }
     }
 
 fapi_try_exit:
@@ -249,9 +286,15 @@ fapi_try_exit:
 fapi2::ReturnCode trigger_csave( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target )
 {
     fapi2::buffer<uint8_t> l_nvdimm_port_bitmap = 0;
+    constexpr uint64_t INVALID = 0;
+    constexpr uint64_t VALID = 1;
 
     FAPI_TRY( FAPI_ATTR_GET(fapi2::ATTR_SBE_NVDIMM_IN_PORT, i_target, l_nvdimm_port_bitmap) );
 
+    // Invalidate the BAR temporarily. This prevents any memory access (proc, IO, etc.) from
+    // remove the port out of STR
+    FAPI_TRY( change_bar_valid_state(i_target, l_nvdimm_port_bitmap, INVALID));
+
     for (uint8_t l_mca_pos = 0; l_mca_pos < PORTS_PER_MODULE; l_mca_pos++)
     {
         if (l_nvdimm_port_bitmap.getBit(l_mca_pos) == fapi2::ENUM_ATTR_SBE_NVDIMM_IN_PORT_YES)
@@ -266,6 +309,10 @@ fapi2::ReturnCode trigger_csave( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHI
         }
     }
 
+    // Set the BAR back to valid. This is a don't care for the power down but required
+    // for MPIPL.
+    FAPI_TRY( change_bar_valid_state(i_target, l_nvdimm_port_bitmap, VALID));
+
 fapi_try_exit:
     return fapi2::current_err;
 }
diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.H b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.H
index 7da4da88..68b9445b 100644
--- a/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.H
+++ b/src/import/chips/p9/procedures/hwp/memory/lib/workarounds/nvdimm_workarounds.H
@@ -65,20 +65,31 @@ fapi2::ReturnCode trigger_csave( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHI
 /// @brief Helper for trigger_csave. This subroutine assert RESET_n to trigger
 /// the backup on nvdimm
 /// @param[in] i_target - PROC_CHIP target
-/// @param[in] l_mca_pos - The MCA position relative to the PROC
+/// @param[in] i_mca_pos - The MCA position relative to the PROC
 /// @return FAPI2_RC_SUCCESS iff setup was successful
 ///
 
-fapi2::ReturnCode assert_resetn( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, uint8_t l_mca_pos);
+fapi2::ReturnCode assert_resetn( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, uint8_t i_mca_pos);
+
+///
+/// @brief change the BAR valid state on MCS with NVDIMM installed
+/// @param[in] i_target - PROC_CHIP target
+/// @param[in] i_port_bitmap - port bitmap that indicates port with nvdimm
+/// @param[in] i_state - BAR state to change to
+/// @return FAPI2_RC_SUCCESS iff setup was successful
+///
+fapi2::ReturnCode change_bar_valid_state( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
+        const fapi2::buffer<uint8_t> i_port_bitmap,
+        const uint64_t i_state);
 
 ///
 /// @brief Helper for trigger_csave. This subroutine puts the MCA into STR
 /// @param[in] i_target - PROC_CHIP target
-/// @param[in] l_mca_pos - The MCA position relative to the PROC
+/// @param[in] i_mca_pos - The MCA position relative to the PROC
 /// @return FAPI2_RC_SUCCESS iff setup was successful
 ///
 
-fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, uint8_t l_mca_pos);
+fapi2::ReturnCode self_refresh_entry( const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, uint8_t i_mca_pos);
 
 }//ns nvdimm
 }//ns workarounds