Run memory training in parallel across centaurs

-Uses a thread pool to do memory training for multiple (up to 4)
 centaurs at a time.
-Also fixes mdiasm.C to set the thread pool count before
 creating a thread pool instead of using whatever was set
 before it.

Change-Id: I7331bfcd0e1e85af1b825e8ac4e0d1d76924014a
RTC: 71239
CQ: SW454787
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/71614
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 148 insertions, 66 deletions

diff --git a/src/usr/isteps/istep13/call_mss_draminit_trainadv.C b/src/usr/isteps/istep13/call_mss_draminit_trainadv.C
index 41e94a076..fc42f7264 100644
--- a/src/usr/isteps/istep13/call_mss_draminit_trainadv.C
+++ b/src/usr/isteps/istep13/call_mss_draminit_trainadv.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -22,12 +22,15 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
+#include <algorithm>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <errl/errludtarget.H>
 #include <isteps/hwpisteperror.H>
 #include <initservice/isteps_trace.H>
 #include <initservice/istepdispatcherif.H>
+#include <util/threadpool.H>
+#include <sys/task.h>
 
 //  targeting support
 #include <targeting/common/commontargeting.H>
@@ -48,13 +51,134 @@ using   namespace   TARGETING;
 
 namespace ISTEP_13
 {
+
+//
+// @brief Mutex to prevent threads from adding details to the step
+//        error log at the same time.
+mutex_t g_stepErrorMutex = MUTEX_INITIALIZER;
+
+/*******************************************************************************
+ * @brief base work item class for isteps (used by thread pool)
+ */
+class IStepWorkItem
+{
+    public:
+        virtual ~IStepWorkItem(){}
+        virtual void operator()() = 0;
+};
+
+/*******************************************************************************
+ * @brief Membuf specific work item class
+ */
+class MembufWorkItem: public IStepWorkItem
+{
+    private:
+        IStepError* iv_pStepError;
+        const TARGETING::Target* iv_pMembuf;
+
+    public:
+        /**
+         * @brief task function, called by threadpool to run the HWP on the
+         *        target
+         */
+         void operator()();
+
+        /**
+         * @brief ctor
+         *
+         * @param[in] i_membuf target membuf to operate on
+         * @param[in] i_istepError error accumulator for this istep
+         */
+        MembufWorkItem(const TARGETING::Target& i_membuf,
+                       IStepError& i_stepError):
+            iv_pStepError(&i_stepError),
+            iv_pMembuf(&i_membuf) {}
+
+        // delete default copy/move constructors and operators
+        MembufWorkItem() = delete;
+        MembufWorkItem(const MembufWorkItem& ) = delete;
+        MembufWorkItem& operator=(const MembufWorkItem& ) = delete;
+        MembufWorkItem(MembufWorkItem&&) = delete;
+        MembufWorkItem& operator=(MembufWorkItem&&) = delete;
+
+        /**
+         * @brief destructor
+         */
+        ~MembufWorkItem(){};
+};
+
+//******************************************************************************
+void MembufWorkItem::operator()()
+{
+    errlHndl_t l_err = nullptr;
+
+    // reset watchdog for each memb as this function can be very slow
+    INITSERVICE::sendProgressCode();
+
+    TARGETING::TargetHandleList l_mbaTargetList;
+    getChildChiplets(l_mbaTargetList,
+                    iv_pMembuf,
+                    TYPE_MBA);
+
+    for (auto l_mbaTarget : l_mbaTargetList)
+    {
+        // Dump current run on target
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+            "Running p9c_mss_draminit_training_advanced HWP on target HUID %.8X.",
+            TARGETING::get_huid(l_mbaTarget));
+
+        //  call the HWP with each target
+        fapi2::Target <fapi2::TARGET_TYPE_MBA_CHIPLET>
+                            l_fapi_mba_target(l_mbaTarget);
+
+        FAPI_INVOKE_HWP(l_err,
+                       p9c_mss_draminit_training_advanced,
+                       l_fapi_mba_target);
+
+        //  process return code.
+        if ( l_err )
+        {
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                "ERROR 0x%.8X : p9c_mss_draminit_training_advanced HWP returns error.",
+                l_err->reasonCode());
+
+            // capture the target data in the elog
+            ErrlUserDetailsTarget(l_mbaTarget).addToLog(l_err);
+
+            //addErrorDetails may not be thread-safe.  Protect with mutex.
+            mutex_lock(&g_stepErrorMutex);
+
+            // Create IStep error log and cross reference to error that occurred
+            iv_pStepError->addErrorDetails( l_err );
+
+            mutex_unlock(&g_stepErrorMutex);
+
+            // Commit Error
+            errlCommit( l_err, HWPF_COMP_ID );
+
+            break;
+        }
+        else
+        {
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                   "SUCCESS running p9c_mss_draminit_training_advanced HWP on target HUID %.8X.",
+                   TARGETING::get_huid(l_mbaTarget));
+        }
+    }
+}
+
+
+//******************************************************************************
 void* call_mss_draminit_trainadv (void *io_pArgs)
 {
-    errlHndl_t l_err = NULL;
+    errlHndl_t l_err = nullptr;
     IStepError l_stepError;
+    Util::ThreadPool<IStepWorkItem> tp;
+    uint32_t l_maxThreads = ISTEP13_MAX_THREADS;
+    uint32_t l_numThreads = 0;
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-            "call_mss_draminit_trainingadv entry" );
+            "call_mss_draminit_trainingadv entry");
 
     // Get all MCBIST targets
     TARGETING::TargetHandleList l_mcbistTargetList;
@@ -100,78 +224,36 @@ void* call_mss_draminit_trainadv (void *io_pArgs)
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Skipping p9c_mss_draminit_training_advanced HWP in Simics");
     }
-    else if(l_stepError.getErrorHandle() == NULL)
+    else if(l_stepError.getErrorHandle() == nullptr)
     {
         // Get all Centaur targets
         TARGETING::TargetHandleList l_membufTargetList;
         getAllChips(l_membufTargetList, TYPE_MEMBUF);
 
-        for (TargetHandleList::const_iterator
-             l_membuf_iter = l_membufTargetList.begin();
-             l_membuf_iter != l_membufTargetList.end();
-             ++l_membuf_iter)
+        for (const auto & l_membuf : l_membufTargetList)
         {
-            // reset watchdog for each memb as this function can be very slow
-            INITSERVICE::sendProgressCode();
-
-            //  make a local copy of the target for ease of use
-            TARGETING::Target* l_pCentaur = *l_membuf_iter;
-
-            TARGETING::TargetHandleList l_mbaTargetList;
-            getChildChiplets(l_mbaTargetList,
-                            l_pCentaur,
-                            TYPE_MBA);
-
-            for (TargetHandleList::const_iterator
-                 l_mba_iter = l_mbaTargetList.begin();
-                 l_mba_iter != l_mbaTargetList.end();
-                 ++l_mba_iter)
-             {
-                //  Make a local copy of the target for ease of use
-                TARGETING::Target*  l_mbaTarget = *l_mba_iter;
-
-                // Dump current run on target
-                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                    "Running p9c_mss_draminit_training_advanced HWP on "
-                    "target HUID %.8X", TARGETING::get_huid(l_mbaTarget));
-
-                //  call the HWP with each target
-                fapi2::Target <fapi2::TARGET_TYPE_MBA_CHIPLET> l_fapi_mba_target(l_mbaTarget);
-
-                FAPI_INVOKE_HWP(l_err, p9c_mss_draminit_training_advanced, l_fapi_mba_target);
-
-                //  process return code.
-                if ( l_err )
-                {
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                        "ERROR 0x%.8X : p9c_mss_draminit_training_advanced HWP returns error",
-                        l_err->reasonCode());
-
-                    // capture the target data in the elog
-                    ErrlUserDetailsTarget(l_mbaTarget).addToLog(l_err);
-
-                    // Create IStep error log and cross reference to error that occurred
-                    l_stepError.addErrorDetails( l_err );
-
-                    // Commit Error
-                    errlCommit( l_err, HWPF_COMP_ID );
- 
-                    break;
-                }
-                else
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                           "SUCCESS running p9c_mss_draminit_training_advanced HWP on "
-                           "target HUID %.8X", TARGETING::get_huid(l_mbaTarget));
-                }
-
-             }
-
+            //  Create a new workitem from this membuf and feed it to the
+            //  thread pool for processing.  Thread pool handles workitem
+            //  cleanup.
+            tp.insert(new MembufWorkItem(*l_membuf, l_stepError));
         }
 
-    }
+        //Don't create more threads than we have targets
+        size_t l_numTargets = l_membufTargetList.size();
+        l_numThreads = std::min((size_t)l_maxThreads, l_numTargets);
+
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                  "Starting %u thread(s) to handle %u membuf target(s)", l_numThreads, l_numTargets);
 
+        //Set the number of threads to use in the threadpool
+        Util::ThreadPoolManager::setThreadCount(l_numThreads);
 
+        //create and start worker threads
+        tp.start();
+
+        //wait for all workitems to complete, then clean up all threads.
+        tp.shutdown();
+    }
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                                 "call_mss_draminit_trainingadv exit" );
diff --git a/src/usr/isteps/istep13/istep13consts.H b/src/usr/isteps/istep13/istep13consts.H
index 0b01a2082..b7e59eb0a 100644
--- a/src/usr/isteps/istep13/istep13consts.H
+++ b/src/usr/isteps/istep13/istep13consts.H
@@ -29,5 +29,5 @@
 const uint8_t UNLIMITED_RUN = 0xFF;
 const uint8_t VPO_NUM_OF_MBAS_TO_RUN = UNLIMITED_RUN;
 const uint8_t VPO_NUM_OF_MEMBUF_TO_RUN = UNLIMITED_RUN;
-
+const uint8_t ISTEP13_MAX_THREADS = 4;
 #endif