Befor flushNVDIMM chip-op make sure all core Instructions are stopped

Change-Id: I5e084e2e12f3886db6db173598dc4122dffdfa2e
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/72238
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>
Reviewed-by: RAJA DAS <rajadas2@in.ibm.com>

6 files changed, 103 insertions, 50 deletions

diff --git a/src/sbefw/app/power/ipl_table.C b/src/sbefw/app/power/ipl_table.C
index d45af9b3..8f31aef6 100644
--- a/src/sbefw/app/power/ipl_table.C
+++ b/src/sbefw/app/power/ipl_table.C
@@ -222,7 +222,7 @@ static istepMap_t g_istepMpiplStartPtrTbl[] =
             // Find all the child cores within proc and call hwp to know the
             // scom state and call instruction control. Also mask spl attention
             // from core.
-            ISTEP_MAP( istepWithCoreState, p9_query_core_access_state ),
+            ISTEP_MAP( istepWithCoreState, NULL ),
             //  Reset the TPM and clear the TPM deconfig bit, it's not a
             //  procedure but local SBE function
             ISTEP_MAP( istepMpiplRstClrTpmBits, NULL ),
@@ -728,51 +728,8 @@ ReturnCode istepWithCoreState( voidfuncptr_t i_hwp)
     #define SBE_FUNC "istepWithCoreState"
     SBE_ENTER(SBE_FUNC);
     ReturnCode l_rc = FAPI2_RC_SUCCESS;
-    Target<TARGET_TYPE_PROC_CHIP > l_procTgt = plat_getChipTarget();
-    for (auto l_coreTgt : l_procTgt.getChildren<fapi2::TARGET_TYPE_CORE>())
-    {
-        bool l_isScanEnable = false;
-        bool l_isCoreScomEnabled = false;
-        SBE_EXEC_HWP(l_rc, reinterpret_cast<sbeIstepHwpCoreScomState_t>( i_hwp ), l_coreTgt,
-                     l_isCoreScomEnabled, l_isScanEnable)
-        if(l_rc != FAPI2_RC_SUCCESS)
-        {
-            SBE_ERROR(SBE_FUNC " p9_query_core_access_state failed, "
-               "RC=[0x%08X]", l_rc);
-            break;
-        }
-        if(l_isCoreScomEnabled) //true
-        {
-            uint8_t l_thread = SMT4_THREAD0;
-            fapi2::buffer<uint64_t> l_data64;
-            uint64_t l_state;
-            bool l_warnCheck = true;
-            do
-            {
-                // Call instruction control stop
-                // TODO RTC 164425 - Can we pass in 1111 i.e. all threads at the
-                // same time instead of individual threads
-                SBE_EXEC_HWP(l_rc,
-                             threadCntlhwp,
-                             l_coreTgt,
-                             (SINGLE_THREAD_BIT_MASK >> l_thread),
-                             PTC_CMD_STOP, l_warnCheck,l_data64, l_state)
-                if(l_rc != FAPI2_RC_SUCCESS)
-                {
-                    SBE_ERROR(SBE_FUNC "p9_thread_control stop Failed for "
-                        "Core Thread  RC[0x%08X]", l_rc);
-                    break;
-                }
-            }while(++l_thread < SMT4_THREAD_MAX);
 
-            l_rc = maskSpecialAttn(l_coreTgt);
-            if( l_rc != FAPI2_RC_SUCCESS)
-            {
-                SBE_ERROR(SBE_FUNC "maskSpecialAttn failed");
-                break;
-            }
-        }
-    }
+    l_rc = stopAllCoreInstructions();
     if( l_rc == FAPI2_RC_SUCCESS )
     {
         l_rc = flushNVDIMM();
@@ -781,6 +738,11 @@ ReturnCode istepWithCoreState( voidfuncptr_t i_hwp)
             SBE_ERROR(SBE_FUNC "flushNVDIMM failed");
         }
     }
+    else
+    {
+        SBE_ERROR(SBE_FUNC "Stop all core instructions is failed, "
+                 "RC=[0x%08X]", l_rc);
+    }
 
     SBE_EXIT(SBE_FUNC);
     return l_rc;
diff --git a/src/sbefw/app/power/sbecmdcntlinst.C b/src/sbefw/app/power/sbecmdcntlinst.C
index e918365e..f5113cfc 100644
--- a/src/sbefw/app/power/sbecmdcntlinst.C
+++ b/src/sbefw/app/power/sbecmdcntlinst.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER sbe Project                                                  */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -30,6 +30,7 @@
  */
 
 #include "sbecmdcntlinst.H"
+#include "sbecmdiplcontrol.H"
 #include "sbefifo.H"
 #include "sbe_sp_intf.H"
 #include "sbetrace.H"
@@ -37,9 +38,14 @@
 
 #include "fapi2.H"
 #include "p9_thread_control.H"
+#include "p9_query_core_access_state.H"
+
 
 using namespace fapi2;
 
+//Utility function to mask special attention
+extern ReturnCode maskSpecialAttn( const Target<TARGET_TYPE_CORE>& i_target);
+
 // TODO via RTC 152424
 // Currently all proecdures in core directory are in seeprom.
 // So we have to use function pointer to force a long call.
@@ -161,6 +167,73 @@ static uint32_t specialWakeUpCoreDeAssert(
 }
 
 ///////////////////////////////////////////////////////////////////////
+// @brief stopAllCoreInstructions  Stop all core instructions function
+//
+// @return  RC from the underlying FIFO utility
+///////////////////////////////////////////////////////////////////////
+ReturnCode stopAllCoreInstructions( )
+{
+    #define SBE_FUNC "stopAllCoreInstructions"
+    SBE_ENTER(SBE_FUNC);
+    ReturnCode l_fapiRc = FAPI2_RC_SUCCESS;
+    Target<TARGET_TYPE_PROC_CHIP > l_procTgt = plat_getChipTarget();
+    for (auto l_coreTgt : l_procTgt.getChildren<fapi2::TARGET_TYPE_CORE>())
+    {
+        bool l_isCoreScomEnabled = false;
+        if(SBE::isSimicsRunning())
+        {
+            l_isCoreScomEnabled = true;
+        }
+        else
+        {
+            bool l_isScanEnable = false;
+            SBE_EXEC_HWP(l_fapiRc, p9_query_core_access_state, l_coreTgt,
+                    l_isCoreScomEnabled, l_isScanEnable)
+            if(l_fapiRc != FAPI2_RC_SUCCESS)
+            {
+                SBE_ERROR(SBE_FUNC " p9_query_core_access_state failed, "
+                        "RC=[0x%08X]", l_fapiRc);
+                break;
+            }
+        }
+        if(l_isCoreScomEnabled) //true
+        {
+            uint8_t l_thread = SMT4_THREAD0;
+            fapi2::buffer<uint64_t> l_data64;
+            uint64_t l_state;
+            bool l_warnCheck = true;
+            do
+            {
+                // Call instruction control stop
+                // TODO RTC 164425 - Can we pass in 1111 i.e. all threads at the
+                // same time instead of individual threads
+                SBE_EXEC_HWP(l_fapiRc,
+                             threadCntlhwp,
+                             l_coreTgt,
+                             (SINGLE_THREAD_BIT_MASK >> l_thread),
+                             PTC_CMD_STOP, l_warnCheck,l_data64, l_state)
+                if(l_fapiRc != FAPI2_RC_SUCCESS)
+                {
+                    SBE_ERROR(SBE_FUNC "p9_thread_control stop Failed for "
+                        "Core Thread  RC[0x%08X]", l_fapiRc);
+                    break;
+                }
+            }while(++l_thread < SMT4_THREAD_MAX);
+
+            l_fapiRc = maskSpecialAttn(l_coreTgt);
+            if( l_fapiRc != FAPI2_RC_SUCCESS)
+            {
+                SBE_ERROR(SBE_FUNC "maskSpecialAttn failed");
+                break;
+            }
+        }
+    }
+    SBE_EXIT(SBE_FUNC);
+    return l_fapiRc;
+    #undef SBE_FUNC
+}
+
+///////////////////////////////////////////////////////////////////////
 // @brief sbeCntlInst Sbe control instructions function
 //
 // @return  RC from the underlying FIFO utility
diff --git a/src/sbefw/app/power/sbecmdcntlinst.H b/src/sbefw/app/power/sbecmdcntlinst.H
index b962d59c..031da3ad 100644
--- a/src/sbefw/app/power/sbecmdcntlinst.H
+++ b/src/sbefw/app/power/sbecmdcntlinst.H
@@ -5,7 +5,8 @@
 /*                                                                        */
 /* OpenPOWER sbe Project                                                  */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2019                        */
+/* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
@@ -32,13 +33,20 @@
 #define __SBEFW_SBECMDCNTLINST_H
 
 #include <stdint.h>
-
+#include "fapi2.H"
 
 // This is used to find out the array index in g_control_reg_map in
 // p9_thread_control.C
 static const uint8_t SINGLE_THREAD_BIT_MASK = 0x08;
 
 /**
+  * @brief stopAllCoreInstructions  Stop all core instructions function
+  *
+  * @return    Rc from the FAPI access utility
+  */
+fapi2::ReturnCode stopAllCoreInstructions( );
+
+/**
   * @brief sbeCntlInst : Implements SBE Control instructions ChipOp
   *
   * @param[in] i_pArg Buffer to be passed to the function (not used as of now)
diff --git a/src/sbefw/app/power/sbecmdflushnvdimm.C b/src/sbefw/app/power/sbecmdflushnvdimm.C
index c551cbca..9159676b 100644
--- a/src/sbefw/app/power/sbecmdflushnvdimm.C
+++ b/src/sbefw/app/power/sbecmdflushnvdimm.C
@@ -34,6 +34,7 @@
 #include "sbeFifoMsgUtils.H"
 #include "nvdimm_workarounds.H"
 #include "sbecmdflushnvdimm.H"
+#include "sbecmdcntlinst.H"
 
 using namespace fapi2;
 
@@ -66,6 +67,15 @@ uint32_t sbeHandleFlushNVDIMM(uint8_t *i_pArg)
         rc = sbeUpFifoDeq_mult (len, NULL);
         CHECK_SBE_RC_AND_BREAK_IF_NOT_SUCCESS(rc);
 
+        fapiRc = stopAllCoreInstructions();
+        if( fapiRc != FAPI2_RC_SUCCESS )
+        {
+            SBE_ERROR(SBE_FUNC "stop all core instructions was failed");
+            respHdr.setStatus( SBE_PRI_GENERIC_EXECUTION_FAILURE,
+                               SBE_SEC_HWP_FAILURE);
+            ffdc.setRc(fapiRc);
+            break;
+        }
         fapiRc = flushNVDIMM();
         if( fapiRc != FAPI2_RC_SUCCESS )
         {
diff --git a/src/test/testcases/test.xml b/src/test/testcases/test.xml
index e7b54cac..23837f6f 100755
--- a/src/test/testcases/test.xml
+++ b/src/test/testcases/test.xml
@@ -36,7 +36,6 @@
           <include>../simics/targets/p9_nimbus/sbeTest/testPSUSetStashPair.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testIstep.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testSuspendIO.xml</include>
-          <include>../simics/targets/p9_nimbus/sbeTest/testFlushNVDIMM.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testGeneric.xml</include>
           <!-- Memory access testcases -->
           <include>../simics/targets/p9_nimbus/sbeTest/testUnsecureMemRegions.xml</include>
@@ -55,6 +54,7 @@
           </testcase>
           <!-- TODO add testCntlInstruction.xml -->
           <include>../simics/targets/p9_nimbus/sbeTest/testRegAccess.xml</include>
+          <include>../simics/targets/p9_nimbus/sbeTest/testFlushNVDIMM.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testFifoReset.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testSystemFabricMap.xml</include>
           <include>../simics/targets/p9_nimbus/sbeTest/testSecurityListDump.xml</include>
diff --git a/src/test/testcases/testFlushNVDIMM.xml b/src/test/testcases/testFlushNVDIMM.xml
index 9effcbaa..072ccf97 100755
--- a/src/test/testcases/testFlushNVDIMM.xml
+++ b/src/test/testcases/testFlushNVDIMM.xml
@@ -6,6 +6,7 @@
 <!-- OpenPOWER sbe Project                                                  -->
 <!--                                                                        -->
 <!-- Contributors Listed Below - COPYRIGHT 2017,2019                        -->
+<!-- [+] International Business Machines Corp.                              -->
 <!--                                                                        -->
 <!--                                                                        -->
 <!-- Licensed under the Apache License, Version 2.0 (the "License");        -->
@@ -27,4 +28,3 @@
           <simcmd>run-python-file targets/p9_nimbus/sbeTest/testFlushNVDIMM.py</simcmd>
           <exitonerror>yes</exitonerror>
       </testcase>
-