Update p9_l2_flush to check if purge is busy on anything prior to flush.

The previous check was only checking once if CMD_REG_BUSY is off.  This
will now check if CMD_PRGSM_BUSY is off, indicating the Purge engine
is not busy with any requests at all.  It will also poll on busy
prior to issuing the flush.

If PURGE_CMD_ERR is on prior to starting flush a warning will be
printed via FAPI_DBG.

PURGE_CMD_ERR will be cleared prior to issuing the purge.

Change-Id: I120cc9a00d26da8cf2ca4ec6dd7d8f3006633b61
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/72562
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: MURULIDHAR NATARAJU <murulidhar@in.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/72582
Reviewed-by: RAJA DAS <rajadas2@in.ibm.com>

3 files changed, 118 insertions, 6 deletions

diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.C b/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.C
index ec447cdb..9aa10514 100755
--- a/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.C
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER sbe Project                                                  */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -58,6 +58,72 @@ enum
 //------------------------------------------------------------------------------
 
 /// See doxygen in header file
+fapi2::ReturnCode purgeReadyCheck(
+    const fapi2::Target<fapi2::TARGET_TYPE_EX>& i_target,
+    const uint64_t i_busyCount,
+    fapi2::buffer<uint64_t>& o_prdPurgeCmdReg)
+{
+    FAPI_DBG("Entering purgeReadyCheck");
+    uint64_t l_loopCount = 0;
+
+    do
+    {
+        FAPI_TRY(fapi2::getScom(i_target, EX_PRD_PURGE_CMD_REG, o_prdPurgeCmdReg),
+                 "Error from getScom EX_PRD_PURGE_CMD_REG");
+
+        // Check the EX_PRD_PURGE_CMD_REG_BUSY bit from scom register
+        if ( !o_prdPurgeCmdReg.getBit(EX_PRD_PURGE_CMD_REG_PRGSM_BUSY))
+        {
+            // PURGE is done, get out
+            break;
+        }
+        else
+        {
+            l_loopCount++;
+
+            if (l_loopCount > i_busyCount)
+            {
+                // Time out, exit loop
+                break;
+            }
+
+            // Delay 10ns for each loop
+            FAPI_TRY(fapi2::delay(P9_L2_FLUSH_HW_NS_DELAY,
+                                  P9_L2_FLUSH_SIM_CYCLE_DELAY),
+                     "Fapi Delay call failed.");
+        }
+    }
+    while (1);
+
+    // Error out if still busy
+    if (l_loopCount > i_busyCount)
+    {
+        // engine busy, dump status
+        FAPI_DBG("Purge engine busy (reg_busy = %d, busy_on_this = %d,"
+                 " sm_busy = %d)",
+                 o_prdPurgeCmdReg.getBit<EX_PRD_PURGE_CMD_REG_BUSY>(),
+                 o_prdPurgeCmdReg.getBit<EX_PRD_PURGE_CMD_REG_PRGSM_BUSY_ON_THIS>(),
+                 o_prdPurgeCmdReg.getBit<EX_PRD_PURGE_CMD_REG_PRGSM_BUSY>());
+
+        FAPI_ASSERT(false, fapi2::P9_PURGE_READY_COMPLETE_TIMEOUT()
+                    .set_TARGET(i_target)
+                    .set_COUNT_THRESHOLD(i_busyCount)
+                    .set_CMD_REG(o_prdPurgeCmdReg),
+                    "Previous purge request has not completed prior to issuing L2 flush.");
+    }
+
+    if (o_prdPurgeCmdReg.getBit<EX_PRD_PURGE_CMD_REG_ERR>())
+    {
+        FAPI_INF("WARNING: EX_PRD_PURGE_CMD_REG_ERR set prior to L2 flush");
+    }
+
+fapi_try_exit:
+    FAPI_DBG("Exiting purgeReadyCheck - Counter: %d; prdPurgeCmdReg: 0x%.16llX",
+             l_loopCount, o_prdPurgeCmdReg);
+    return fapi2::current_err;
+}
+
+/// See doxygen in header file
 fapi2::ReturnCode purgeCompleteCheck(
     const fapi2::Target<fapi2::TARGET_TYPE_EX>& i_target,
     const uint64_t i_busyCount,
@@ -79,7 +145,7 @@ fapi2::ReturnCode purgeCompleteCheck(
                     "Purge failed. EX_PRD_PURGE_CMD_REG_ERR set");
 
         // Check the EX_PRD_PURGE_CMD_REG_BUSY bit from scom register
-        if ( !o_prdPurgeCmdReg.getBit(EX_PRD_PURGE_CMD_REG_BUSY) )
+        if ( !o_prdPurgeCmdReg.getBit(EX_PRD_PURGE_CMD_REG_BUSY))
         {
             // PURGE is done, get out
             break;
@@ -140,6 +206,7 @@ fapi2::ReturnCode setupAndTriggerPrdPurge(
     // ensure PURGE_CMD_TYPE/MEM/CGC/BANK are clear to specify flush
     // of entire cache
     FAPI_DBG("Write L2 Purge Engine Command Register to initiate cache flush");
+    l_cmdReg.clearBit<EX_PRD_PURGE_CMD_REG_ERR>();
     l_cmdReg.insert<EX_PRD_PURGE_CMD_REG_TYPE,
                     EX_PRD_PURGE_CMD_REG_TYPE_LEN>(i_purgeData.iv_cmdType);
     l_cmdReg.insert<EX_PRD_PURGE_CMD_REG_MEM,
@@ -176,8 +243,8 @@ fapi2::ReturnCode l2_flush_start(
 
     // Ensure that purge engine is idle before starting flush
     // poll Purge Engine status
-    FAPI_TRY(purgeCompleteCheck(i_target, 0, l_cmdReg), // 0 = no wait
-             "Error returned from purgeCompleteCheck call");
+    FAPI_TRY(purgeReadyCheck(i_target, P9_L2_FLUSH_MAX_POLLS, l_cmdReg), // 0 = no wait
+             "Error returned from purgeReadyCheck call");
 
     FAPI_TRY(setupAndTriggerPrdPurge(i_target, i_purgeData, l_cmdReg),
              "Error returned from setupAndTriggerPrdPurge");
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.H b/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.H
index af02fccd..ae8488e2 100755
--- a/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.H
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_l2_flush.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER sbe Project                                                  */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -93,6 +93,23 @@ typedef fapi2::ReturnCode (*p9_l2_flush_FP_t)
 extern "C"
 {
 
+///-----------------------------------------------------------------------------
+///
+/// @brief  Check if any purge is in progress before issuing a new one.
+///
+/// @param[in]  i_target          => EX chiplet target
+/// @param[in]  i_busyCount       => Max busy count waiting for idle
+/// @param[out] o_prdPurgeCmdReg  => EX_PRD_PURGE_CMD_REG value.
+///
+/// @return FAPI2_RC_SUCCESS if engine status returns as idle (with no errors)
+///         before maximum number of polls has been reached
+///         else, return error.
+    fapi2::ReturnCode purgeReadyCheck(
+        const fapi2::Target<fapi2::TARGET_TYPE_EX>& i_target,
+        const uint64_t i_busyCount,
+        fapi2::buffer<uint64_t>& o_prdPurgeCmdReg);
+
+///-----------------------------------------------------------------------------
 ///
 /// @brief  Utility function to check for a purge operation to be completed.
 ///         This function polls the EX_PRD_PURGE_CMD_REG_BUSY bit of
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_l2_flush_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_l2_flush_errors.xml
index 62f474c2..a8851fff 100755
--- a/src/import/chips/p9/procedures/xml/error_info/p9_l2_flush_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_l2_flush_errors.xml
@@ -5,7 +5,7 @@
 <!--                                                                        -->
 <!-- OpenPOWER sbe Project                                                  -->
 <!--                                                                        -->
-<!-- Contributors Listed Below - COPYRIGHT 2015,2017                        -->
+<!-- Contributors Listed Below - COPYRIGHT 2015,2019                        -->
 <!-- [+] International Business Machines Corp.                              -->
 <!--                                                                        -->
 <!--                                                                        -->
@@ -100,5 +100,33 @@
     </callout>
   </hwpError>
   <!-- ********************************************************************* -->
+  <hwpError>
+    <rc>RC_P9_PURGE_READY_COMPLETE_TIMEOUT</rc>
+    <ffdc>TARGET</ffdc>
+    <ffdc>CMD_REG</ffdc>
+    <ffdc>COUNT_THRESHOLD</ffdc>
+    <description>
+      Procedure: p9_l2_flush
+      Timed out waiting for purge busy indication to clear in L2 Purge Engine
+      Prior to issuing the L2 flush via register.
+    </description>
+    <collectRegisterFfdc>
+      <id>REG_FFDC_PROC_L2_REGISTERS</id>
+      <target>TARGET</target>
+      <targetType>TARGET_TYPE_PROC_CHIP</targetType>
+    </collectRegisterFfdc>
+    <callout>
+      <target>TARGET</target>
+      <priority>HIGH</priority>
+    </callout>
+    <deconfigure>
+      <target>TARGET</target>
+    </deconfigure>
+    <callout>
+      <procedure>CODE</procedure>
+      <priority>LOW</priority>
+    </callout>
+  </hwpError>
+  <!-- ********************************************************************* -->
 
 </hwpErrors>