Add Child RC Checking to Thread Pool

Add logic for Thread Pool to return error log when
shutdown() is called if a child thread (or multiple
child threads) had crashed. This is a default behavior,
but a constructor option can switch this functionality
off.

The new logic will aid in debugging of the failures of
child processes, as the failures could be catched when
they occurred and not at some point after.

Change-Id: I9736d659a086701b8e4f18f41504df4864924d88
RTC: 208517
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/75897
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

3 files changed, 29 insertions, 7 deletions

diff --git a/src/include/usr/util/impl/threadpool.H b/src/include/usr/util/impl/threadpool.H
index 84ee9afd7..5ad530057 100644
--- a/src/include/usr/util/impl/threadpool.H
+++ b/src/include/usr/util/impl/threadpool.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2019                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -38,6 +40,7 @@
 #include <algorithm>
 #include <sys/sync.h>
 #include <sys/task.h>
+#include <errl/errlentry.H>
 
 namespace Util
 {
@@ -82,7 +85,11 @@ namespace __Util_ThreadPool_Impl
 
                 /** Simple constructor, call __init to avoid the in-charge and
                  *  not-in-charge construction costs. */
-            ThreadPoolImpl() { __init(); };
+            ThreadPoolImpl(bool i_checkChildRc = true) :
+                    iv_checkChildRc(i_checkChildRc)
+            {
+                __init();
+            };
 
         protected:
                 /** Initialize the object. */
@@ -105,7 +112,7 @@ namespace __Util_ThreadPool_Impl
                  */
             void __start(start_fn_t fn, void* instance);
                 /** Stop the thread-pool. */
-            void __shutdown();
+            errlHndl_t __shutdown();
 
                 /** Outstanding work-list. */
             worklist_t iv_worklist;
@@ -119,6 +126,7 @@ namespace __Util_ThreadPool_Impl
             std::list<tid_t> iv_children;
                 /** State of object. */
             bool iv_shutdown;
+            bool iv_checkChildRc;
 
     };
 
diff --git a/src/include/usr/util/threadpool.H b/src/include/usr/util/threadpool.H
index f96e0b916..43fdeba7a 100644
--- a/src/include/usr/util/threadpool.H
+++ b/src/include/usr/util/threadpool.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -58,6 +58,7 @@
 #include <stdint.h>
 #include <util/traits/has_lessthan.H>
 #include <util/impl/threadpool.H>
+#include <errl/errlentry.H>
 
 namespace Util
 {
@@ -87,7 +88,10 @@ class ThreadPool : public Util::__Util_ThreadPool_Impl::ThreadPoolImpl
 {
     public:
             /** Basic Constructor.  Initialize ThreadPool. */
-        ThreadPool() : Util::__Util_ThreadPool_Impl::ThreadPoolImpl() { };
+        ThreadPool(bool i_checkChildRc = true) :
+                Util::__Util_ThreadPool_Impl::ThreadPoolImpl(i_checkChildRc)
+        {
+        };
             /** Basic Destructor.  Ensures pool is properly shut down. */
         ~ThreadPool() { shutdown(); };
 
@@ -98,12 +102,20 @@ class ThreadPool : public Util::__Util_ThreadPool_Impl::ThreadPoolImpl
                 __start(reinterpret_cast<start_fn_t>(&run), this);
             };
             /** @brief Completes outstanding work and destroys worker threads.
+             *         Returns an error log when any child task crashes when
+             *         iv_checkChildRc is set.
              *
              *  @note This function will block until all work is completed and
              *        worker threads are destroyed.
+             *
+             * @return nullptr if all child tasks finished (child status
+             *         checking disabled);
+             *         nullptr if all child tasks finished successfully, or a
+             *         pointer to error log otherwise (child status checking
+             *         enabled).
              */
-        void shutdown()
-            { __shutdown(); };
+        errlHndl_t shutdown()
+            { return __shutdown(); };
 
             /** @brief Insert a work item onto the thread-pool's queue.
              *
diff --git a/src/include/usr/util/util_reasoncodes.H b/src/include/usr/util/util_reasoncodes.H
index 17ea0edfc..1f8146baf 100644
--- a/src/include/usr/util/util_reasoncodes.H
+++ b/src/include/usr/util/util_reasoncodes.H
@@ -53,6 +53,7 @@ namespace Util
         UTIL_MOD_GET_OBUS_PLL_BUCKET = 0x14,  // UtilCommonAttr::getObusPllBucket
         UTIL_LIDMGR_CSTOR       = 0x15,       // UtilLidMgr::UtilLidMgr
         UTIL_MCL_PROCESS_SINGLE_COMP = 0x16,  // UtilLidMgr::processSingleComponent
+        UTIL_MOD_TP_SHUTDOWN    = 0x17,       // Util::__Util_ThreadPool_Impl::ThreadPoolImpl::__shutdown
     };
 
     enum ReasonCode
@@ -85,6 +86,7 @@ namespace Util
         UTIL_ERC_NO_MATCHING_FREQ        = UTIL_COMP_ID | 0x1B,
         UTIL_LIDMGR_INVAL_LID_REQUEST    = UTIL_COMP_ID | 0x1C,
         UTIL_LIDMGR_INVAL_COMP           = UTIL_COMP_ID | 0x1D,
+        UTIL_RC_CHILD_TASK_FAILED        = UTIL_COMP_ID | 0x1E,
     };
 };