Use thread_local for globals that require it

Change-Id: I84294a60c53da2501b7bb1b53fe84d9f9709f8b1
RTC:147599
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/72379
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

5 files changed, 41 insertions, 49 deletions

diff --git a/src/build/mkrules/cflags.env.mk b/src/build/mkrules/cflags.env.mk
index df40f77ff..732e6559e 100644
--- a/src/build/mkrules/cflags.env.mk
+++ b/src/build/mkrules/cflags.env.mk
@@ -34,7 +34,7 @@ ifdef MODULE
 COMMONFLAGS += -fPIC -Bsymbolic -Bsymbolic-functions
 CFLAGS += -D__HOSTBOOT_MODULE=$(MODULE)
 CFLAGS += -DNO_INITIALIZER_LIST
-CFLAGS += -DPLAT_NO_THREAD_LOCAL_STORAGE
+#CFLAGS += -DPLAT_NO_THREAD_LOCAL_STORAGE
 CFLAGS += -D__FAPI
 endif
 
diff --git a/src/include/usr/fapi2/plat_hw_access.H b/src/include/usr/fapi2/plat_hw_access.H
index 0d4e56a2b..1ce681f61 100644
--- a/src/include/usr/fapi2/plat_hw_access.H
+++ b/src/include/usr/fapi2/plat_hw_access.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -278,7 +278,11 @@ void platSetOpMode(const OpModes i_mode);
 /// @return the operational mode
 OpModes platGetOpMode(void);
 
+#ifndef PLAT_NO_THREAD_LOCAL_STORAGE
+extern thread_local OpModes opMode;
+#else
 extern OpModes opMode;
+#endif
 
 //--------------------------------------------------------------------------
 // PIB Error Mask Functions
diff --git a/src/include/usr/fapi2/plat_hwp_invoker.H b/src/include/usr/fapi2/plat_hwp_invoker.H
index 894cc1927..0eace34bc 100644
--- a/src/include/usr/fapi2/plat_hwp_invoker.H
+++ b/src/include/usr/fapi2/plat_hwp_invoker.H
@@ -45,12 +45,10 @@
 #include <xscom/piberror.H>
 
 namespace fapi2 {
-//@fixme-RTC:147599-Remove when thread-local storage works right
-///
-/// @brief Lock or unlock the HWP futex
-/// @param[i] i_lock  true:lock the mutex, false:unlock
-///
-void hwpLock( bool i_lock );
+/*
+ * @brief Resets all HWP thread_local vars
+ */
+void hwpResetGlobals(void);
 }
 
 /**
@@ -68,7 +66,7 @@ void hwpLock( bool i_lock );
 
 #define FAPI_INVOKE_HWP(ERRHNDL, FUNC, _args_...) \
     {\
-        fapi2::hwpLock(true); \
+        fapi2::hwpResetGlobals(); \
         fapi2::ReturnCode invoke_rc; \
         FAPI_EXEC_HWP(invoke_rc, FUNC, ##_args_); \
         ERRHNDL = fapi2::rcToErrl(invoke_rc);\
@@ -76,7 +74,7 @@ void hwpLock( bool i_lock );
             ERRHNDL->collectTrace(FAPI_IMP_TRACE_NAME,256);\
             ERRHNDL->collectTrace(FAPI_TRACE_NAME,384);\
         }\
-        fapi2::hwpLock(false); \
+        fapi2::hwpResetGlobals(); \
     }
 
 /**
@@ -95,14 +93,14 @@ void hwpLock( bool i_lock );
 
 #define FAPI_INVOKE_HWP_RC(ERRHNDL, _RC, FUNC, _args_...) \
     {\
-        fapi2::hwpLock(true); \
+        fapi2::hwpResetGlobals(); \
         FAPI_EXEC_HWP(_RC, FUNC, ##_args_); \
         ERRHNDL = fapi2::rcToErrl(_RC);\
         if( ERRHNDL ) {\
             ERRHNDL->collectTrace(FAPI_IMP_TRACE_NAME,256);\
             ERRHNDL->collectTrace(FAPI_TRACE_NAME,384);\
         }\
-        fapi2::hwpLock(false); \
+        fapi2::hwpResetGlobals(); \
     }
 
 /**
@@ -117,9 +115,9 @@ void hwpLock( bool i_lock );
  */
 #define FAPI_CALL_HWP( RETURN, FUNC, _args_...) \
    {\
-        fapi2::hwpLock(true); \
+        fapi2::hwpResetGlobals(); \
         RETURN = FUNC(_args_); \
-        fapi2::hwpLock(false); \
+        fapi2::hwpResetGlobals(); \
    }
 
 /**
@@ -133,9 +131,9 @@ void hwpLock( bool i_lock );
  */
 #define FAPI_CALL_HWP_NORETURN( FUNC, _args_...) \
    {\
-        fapi2::hwpLock(true); \
+        fapi2::hwpResetGlobals(); \
         FUNC(_args_); \
-        fapi2::hwpLock(false); \
+        fapi2::hwpResetGlobals(); \
    }
 
 
diff --git a/src/usr/fapi2/plat_hw_access.C b/src/usr/fapi2/plat_hw_access.C
index 9339e4ed9..dad30b177 100644
--- a/src/usr/fapi2/plat_hw_access.C
+++ b/src/usr/fapi2/plat_hw_access.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -58,11 +58,20 @@ const uint32_t CFAM_ENGINE_OFFSET = 0xFE00;
 // Function prototypes
 uint64_t platGetDDScanMode(const uint32_t i_ringMode);
 
-//TODO RTC:147599 Make this thread_local
+// Operational mode for scom operations (ignore errors, wakeup core, etc.)
+#ifndef PLAT_NO_THREAD_LOCAL_STORAGE
+thread_local OpModes opMode = NORMAL;
+#else
 OpModes opMode = NORMAL;
+#endif
 
-//TODO RTC:147599 Make pib_err_mask thread local
+// Bitmap of PIB errors to ignore during a PIB oeration
+#ifndef PLAT_NO_THREAD_LOCAL_STORAGE
+thread_local uint8_t pib_err_mask = 0x00;
+#else
 uint8_t pib_err_mask = 0x00;
+#endif
+
 
 //------------------------------------------------------------------------------
 // HW Communication Functions to be implemented at the platform layer.
diff --git a/src/usr/fapi2/plat_utils.C b/src/usr/fapi2/plat_utils.C
index ee1fde715..01eacb645 100644
--- a/src/usr/fapi2/plat_utils.C
+++ b/src/usr/fapi2/plat_utils.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -70,11 +70,12 @@ TRAC_INIT(&g_fapiMfgTd, FAPI_MFG_TRACE_NAME, 4*KILOBYTE);
 namespace fapi2
 {
 
-//@fixme-RTC:147599
 // Define global current_err
-//thread_local ReturnCode current_err;
+#ifndef PLAT_NO_THREAD_LOCAL_STORAGE
+thread_local ReturnCode current_err;
+#else
 ReturnCode current_err;
-
+#endif
 
 ///
 /// @brief Retrieve the ring data from the centaur hw image
@@ -1516,35 +1517,15 @@ fapi2::ReturnCode platSpecialWakeup(const Target<TARGET_TYPE_ALL>& i_target,
     return fapi_rc;
 }
 
-//@fixme-RTC:147599-Remove when thread-local storage works right
-///
-/// @brief Mutex to prevent multiple threads from running HWPs at the same time
 ///
-mutex_t g_fapi2Mux = MUTEX_INITIALIZER;
-
-//@fixme-RTC:147599-Remove when thread-local storage works right
+/// @brief Resets all HWP thread_local variables
 ///
-/// @brief Lock or unlock the HWP futex
-/// @param[i] i_lock  true:lock the mutex, false:unlock
-///
-void hwpLock( bool i_lock )
+void hwpResetGlobals(void)
 {
-    if( i_lock )
-    {
-        mutex_lock(&g_fapi2Mux);
-        // Clear out all of our global (fake TLS) vars before we start
-        fapi2::current_err = fapi2::FAPI2_RC_SUCCESS;
-        fapi2::opMode = fapi2::NORMAL;
-        fapi2::setPIBErrorMask(0);
-    }
-    else
-    {
-        fapi2::current_err = fapi2::FAPI2_RC_SUCCESS;
-        fapi2::opMode = fapi2::NORMAL;
-        fapi2::setPIBErrorMask(0);
-        // Clear out all of our global (fake TLS) vars after we finish
-        mutex_unlock(&g_fapi2Mux);
-    }
+    // Reset all HWP thread_local vars
+    fapi2::current_err = fapi2::FAPI2_RC_SUCCESS;
+    fapi2::opMode = fapi2::NORMAL;
+    fapi2::setPIBErrorMask(0);
 }