Enforce single-threaded rule for Hardware Procedures

Because we aren't using read thread-local storage for the FAPI
variables (opmode, piberrmask, current_err) we need to ensure
that we never have multiple HWPs running at the same time.  The
external interface that we use in all cases is FAPI_INVOKE_HWP so
that is where a mutex is placed.

This change also uncovered a couple bugs in how we were executing
some non-fapi HWPs in the SBE update code so I fixed those as well.

Change-Id: Ie8817da62dd4e6bc9ed3ac2debf126f6d05c2b23
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/34518
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Reviewed-by: Richard J. Knight <rjknight@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 143 insertions, 112 deletions

diff --git a/src/usr/fapi2/plat_utils.C b/src/usr/fapi2/plat_utils.C
index b0fe05181..b95f3c76f 100644
--- a/src/usr/fapi2/plat_utils.C
+++ b/src/usr/fapi2/plat_utils.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -37,6 +37,7 @@
 #include <error_info.H>
 #include <assert.h>
 #include <plat_utils.H>
+#include <hw_access.H>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <hwpf_fapi2_reasoncodes.H>
@@ -66,9 +67,11 @@ TRAC_INIT(&g_fapiMfgTd, FAPI_MFG_TRACE_NAME, 4*KILOBYTE);
 namespace fapi2
 {
 
+//@fixme-RTC:147599
 // Define global current_err
 //thread_local ReturnCode current_err;
 ReturnCode current_err;
+
 ///
 /// @brief Translates a FAPI callout priority to an HWAS callout priority
 ///
@@ -850,6 +853,8 @@ errlHndl_t rcToErrl(ReturnCode & io_rc,
                                                MOD_FAPI2_RC_TO_ERRL,
                                                RC_HWP_GENERATED_ERROR,
                                                l_rcValue);
+            // Note - If location of RC value changes, must update
+            //   ErrlEntry::getFapiRC accordingly
 
             // Add the rcValue as FFDC. This will explain what the error was
             l_pError->addFFDC(HWPF_COMP_ID, &l_rcValue, sizeof(l_rcValue), 1,
@@ -1041,4 +1046,36 @@ fapi2::ReturnCode platSpecialWakeup(const Target<TARGET_TYPE_ALL>& i_target,
     return fapi_rc;
 }
 
+//@fixme-RTC:147599-Remove when thread-local storage works right
+///
+/// @brief Mutex to prevent multiple threads from running HWPs at the same time
+///
+mutex_t g_fapi2Mux = MUTEX_INITIALIZER;
+
+//@fixme-RTC:147599-Remove when thread-local storage works right
+///
+/// @brief Lock or unlock the HWP futex
+/// @param[i] i_lock  true:lock the mutex, false:unlock
+///
+void hwpLock( bool i_lock )
+{
+    if( i_lock )
+    {
+        mutex_lock(&g_fapi2Mux);
+        // Clear out all of our global (fake TLS) vars before we start
+        fapi2::current_err = fapi2::FAPI2_RC_SUCCESS;
+        fapi2::opMode = fapi2::NORMAL;
+        fapi2::setPIBErrorMask(0);
+    }
+    else
+    {
+        fapi2::current_err = fapi2::FAPI2_RC_SUCCESS;
+        fapi2::opMode = fapi2::NORMAL;
+        fapi2::setPIBErrorMask(0);
+        // Clear out all of our global (fake TLS) vars after we finish
+        mutex_unlock(&g_fapi2Mux);
+    }
+}
+
+
 } //end namespace
diff --git a/src/usr/fapi2/test/fapi2HwAccessTest.H b/src/usr/fapi2/test/fapi2HwAccessTest.H
index c113f5d18..867f5ac54 100644
--- a/src/usr/fapi2/test/fapi2HwAccessTest.H
+++ b/src/usr/fapi2/test/fapi2HwAccessTest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -49,6 +49,12 @@
 
 using namespace fapi2;
 
+//This function does nothing, it is used to call FAPI_INVOKE on
+fapi2::ReturnCode empty_function(void)
+{
+    return fapi2::current_err;
+}
+
 
 class Fapi2HwAccessTest : public CxxTest::TestSuite
 {
@@ -191,16 +197,6 @@ void test_fapi2HwAccess()
         }
 
         numTests++;
-        FAPI_INVOKE_HWP(l_errl, p9_opmodetest_getsetopmode);
-        if(l_errl)
-        {
-            TS_FAIL("p9_opmodetest_getsetopmode !!");
-            numFails++;
-            delete l_errl; // delete unexpected error log so we dont get
-                           // a false negative on the next case
-        }
-
-        numTests++;
         uint8_t failed = 0;
         FAPI_INVOKE_HWP(l_errl,
                         p9_opmodetest_ignorehwerr,
@@ -218,15 +214,6 @@ void test_fapi2HwAccess()
         }
 
         numTests++;
-        FAPI_INVOKE_HWP(l_errl, p9_piberrmask_getsettest);
-        if(l_errl)
-        {
-            TS_FAIL("p9_piberrmask_getsettest returned an error!");
-            numFails++;
-            errlCommit(l_errl,FAPI2_COMP_ID);
-        }
-
-        numTests++;
         FAPI_INVOKE_HWP(l_errl, p9_piberrmask_masktest, fapi2_procTarget);
         if(l_errl)
         {
@@ -275,6 +262,102 @@ void test_fapi2HwAccess()
     FAPI_INF("fapi2HwAccessTest Test Complete. %d/%d fails",  numFails , numTests);
 }
 
+void test_piberrmask()
+{
+    FAPI_INF("Entering test_piberrmask...");
+    FAPI_INF("Ensure that getPIBErrorMask return 0 initially");
+
+    errlHndl_t l_errl = NULL;
+
+    do
+    {
+        uint8_t mask = fapi2::getPIBErrorMask();
+        if(mask != 0)
+        {
+            TS_FAIL("test_piberrmask>> Expected fapi2::getPIBErrorMask to return (0x0) but instead returned 0x%x", mask);
+            break;
+        }
+
+        FAPI_INF("Setting pib_err_mask to PIB_CHIPLET_OFFLINE (0x2) and checking that we get it back with getPIBErrorMask");
+
+        fapi2::setPIBErrorMask((uint8_t)PIB::PIB_CHIPLET_OFFLINE);
+        mask = fapi2::getPIBErrorMask();
+        if(mask != PIB::PIB_CHIPLET_OFFLINE)
+        {
+            TS_FAIL("test_piberrmask>> Expected fapi2::getPIBErrorMask to return 0x2 but instead returned 0x%x", mask);
+            break;
+        }
+
+        //Call FAPI_INVOKE on an empty function to test if
+        //it resets the pib err mask
+        FAPI_INVOKE_HWP(l_errl,empty_function);
+        if( l_errl )
+        {
+            TS_FAIL("test_piberrmask got an error !!");
+            delete l_errl; // delete unexpected error log so we dont get
+                           // a false negative on the next case
+        }
+
+        mask = fapi2::getPIBErrorMask();
+        if(mask != 0)
+        {
+            TS_FAIL("test_piberrmask>> Expected fapi2::getPIBErrorMask to return PIB_NO_ERROR (0x0) but instead returned %x , FAPI_INVOKE failed to reset pib_err_mask", mask);
+            break;
+        }
+
+    }while(0);
+
+    FAPI_INF("Exiting test_piberrmask...");
+}
+
+
+void test_getsetopmode()
+{
+    FAPI_INF("Enter test_getsetopmode");
+    FAPI_INF("Ensure that getOpMode return NORMAL initially");
+
+    do
+    {
+        fapi2::OpModes mode = fapi2::getOpMode();
+        if(mode != fapi2::NORMAL)
+        {
+            TS_FAIL("test_getsetopmode>> Expected fapi2::getOpMode to return fapi2::NORMAL (0x0) but instead returned %x", mode);
+            break;
+        }
+
+        FAPI_INF("Setting opMode to IGNORE_HW_ERROR (0x1) and checking that we get it back with getOpMode");
+
+        fapi2::setOpMode(fapi2::IGNORE_HW_ERROR);
+        mode = fapi2::getOpMode();
+        if(mode != fapi2::IGNORE_HW_ERROR)
+        {
+            TS_FAIL("test_getsetopmode>> Expected fapi2::getOpMode to return fapi2::IGNORE_HW_ERROR (0x1) but instead returned %x", mode);
+            break;
+        }
+
+        //Call FAPI_INVOKE on an empty function to test if it resets the opMode
+        errlHndl_t l_errl = NULL;
+        FAPI_INVOKE_HWP(l_errl,empty_function);
+        if( l_errl )
+        {
+            TS_FAIL("test_getsetopmode got an error !!");
+            delete l_errl; // delete unexpected error log so we dont get
+                           // a false negative on the next case
+        }
+
+        mode = fapi2::getOpMode();
+        if(mode != fapi2::NORMAL)
+        {
+            TS_FAIL("test_getsetopmode>> Expected fapi2::getOpMode to return fapi2::NORMAL (0x0) but instead returned %x , FAPI_INVOKE failed to reset opmode", mode);
+            break;
+        }
+
+    }while(0);
+
+    FAPI_INF("Exiting test_getsetopmode");
+}
+
+
 };
 
 #endif // End __FAPI2_HWACCESSTEST_H
diff --git a/src/usr/fapi2/test/p9_hwtests.C b/src/usr/fapi2/test/p9_hwtests.C
index fedf8582f..1f8a713ec 100644
--- a/src/usr/fapi2/test/p9_hwtests.C
+++ b/src/usr/fapi2/test/p9_hwtests.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -35,12 +35,6 @@
 #include <xscom/piberror.H>
 #include <plat_hwp_invoker.H>
 
-//This function does nothing, it is used to call FAPI_INVOKE on
-fapi2::ReturnCode empty_function(void)
-{
-    return fapi2::current_err;
-}
-
 fapi2::ReturnCode p9_scomtest_getscom_fail(
                fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target)
 {
@@ -325,86 +319,6 @@ fapi2::ReturnCode p9_platPutRingWRingID_pass()
 }
 
 
-fapi2::ReturnCode p9_opmodetest_getsetopmode()
-{
-    FAPI_INF("Ensure that getOpMode return NORMAL initially");
-    do
-    {
-        fapi2::OpModes mode = fapi2::getOpMode();
-        if(mode != fapi2::NORMAL)
-        {
-            TS_FAIL("p9_opmodetest_getsetopmode>> Expected fapi2::getOpMode to return fapi2::NORMAL (0x0) but instead returned %x", mode);
-            break;
-        }
-
-        FAPI_INF("Setting opMode to IGNORE_HW_ERROR (0x1) and checking that we get it back with getOpMode");
-
-        fapi2::setOpMode(fapi2::IGNORE_HW_ERROR);
-        mode = fapi2::getOpMode();
-        if(mode != fapi2::IGNORE_HW_ERROR)
-        {
-            TS_FAIL("p9_opmodetest_getsetopmode>> Expected fapi2::getOpMode to return fapi2::IGNORE_HW_ERROR (0x1) but instead returned %x", mode);
-            break;
-        }
-
-        //Call FAPI_INVOKE on an empty function to test if it resets the opMode
-        errlHndl_t l_errl = NULL;
-        FAPI_INVOKE_HWP(l_errl,empty_function);
-
-        mode = fapi2::getOpMode();
-        if(mode != fapi2::NORMAL)
-        {
-            TS_FAIL("p9_opmodetest_getsetopmode>> Expected fapi2::getOpMode to return fapi2::NORMAL (0x0) but instead returned %x , FAPI_INVOKE failed to reset opmode", mode);
-            break;
-        }
-
-    }while(0);
-    return fapi2::current_err;
-}
-
-fapi2::ReturnCode p9_piberrmask_getsettest()
-{
-    FAPI_INF("Entering p9_piberrmask_getsettest...");
-
-    FAPI_INF("Ensure that getPIBErrorMask return 0 initially");
-
-    uint8_t mask = fapi2::getPIBErrorMask();
-    do
-    {
-        if(mask != 0)
-        {
-            TS_FAIL("p9_piberrmask_getsettest>> Expected fapi2::getPIBErrorMask to return (0x0) but instead returned 0x%x", mask);
-            break;
-        }
-
-        FAPI_INF("Setting pib_err_mask to PIB_CHIPLET_OFFLINE (0x2) and checking that we get it back with getPIBErrorMask");
-
-        fapi2::setPIBErrorMask((uint8_t)PIB::PIB_CHIPLET_OFFLINE);
-        mask = fapi2::getPIBErrorMask();
-        if(mask != PIB::PIB_CHIPLET_OFFLINE)
-        {
-            TS_FAIL("p9_piberrmask_getsettest>> Expected fapi2::getPIBErrorMask to return 0x2 but instead returned 0x%x", mask);
-            break;
-        }
-
-        //Call FAPI_INVOKE on an empty function to test if
-        //it resets the pib err mask
-        errlHndl_t l_errl = NULL;
-        FAPI_INVOKE_HWP(l_errl,empty_function);
-
-        mask = fapi2::getPIBErrorMask();
-        if(mask != 0)
-        {
-            TS_FAIL("p9_piberrmask_getsettest>> Expected fapi2::getPIBErrorMask to return PIB_NO_ERROR (0x0) but instead returned %x , FAPI_INVOKE failed to reset pib_err_mask", mask);
-            break;
-        }
-
-    FAPI_INF("Exiting p9_piberrmask_getsettest...");
-    }while(0);
-
-    return fapi2::current_err;
-}
-
 fapi2::ReturnCode p9_opmodetest_ignorehwerr(
                 fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                 uint8_t o_fail)
diff --git a/src/usr/fapi2/test/p9_hwtests.H b/src/usr/fapi2/test/p9_hwtests.H
index cdafe1b06..984c9fce3 100644
--- a/src/usr/fapi2/test/p9_hwtests.H
+++ b/src/usr/fapi2/test/p9_hwtests.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -72,14 +72,11 @@ fapi2::ReturnCode p9_cfamtest_getcfam_pass(
 fapi2::ReturnCode p9_cfamtest_putcfam_pass(
                fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target);
 
-fapi2::ReturnCode p9_opmodetest_getsetopmode();
-
 fapi2::ReturnCode p9_platPutRingWRingID_pass();
 
 fapi2::ReturnCode p9_opmodetest_ignorehwerr(
                 fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
                 uint8_t fail);
-fapi2::ReturnCode p9_piberrmask_getsettest();
 
 fapi2::ReturnCode p9_piberrmask_masktest(
                 fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target);