Improve stability of the runtime in parent/child processes

This change improves stability of the runtime when the application forks child
processes.  Acquiring/releasing __kmp_initz_lock and __kmp_forkjoin_lock in the
atfork handlers insures that the actual fork does not occur while those two
locks are held, and __kmp_itt_reset() reverts the itt's global state to the
initial state which also initializes the mutex stored in the global state.
Some missing initialization code was also inserted in the child's atfork handler.

Patch by Hansang Bae

Differential Revision: https://reviews.llvm.org/D41462

llvm-svn: 322202

6 files changed, 32 insertions, 4 deletions

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index eaa4be540d7..b36bbeaf383 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2788,6 +2788,7 @@ extern char const *__kmp_barrier_pattern_name[bp_last_bar];
 /* Global Locks */
 extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */
 extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
+extern kmp_bootstrap_lock_t __kmp_task_team_lock;
 extern kmp_bootstrap_lock_t
     __kmp_exit_lock; /* exit() is not always thread-safe */
 #if KMP_USE_MONITOR
@@ -2976,6 +2977,7 @@ extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */
 /* read/write: lock */
 extern volatile kmp_team_t *__kmp_team_pool;
 extern volatile kmp_info_t *__kmp_thread_pool;
+extern kmp_info_t *__kmp_thread_pool_insert_pt;
 
 // total num threads reachable from some root thread including all root threads
 extern volatile int __kmp_nth;
diff --git a/openmp/runtime/src/kmp_itt.cpp b/openmp/runtime/src/kmp_itt.cpp
index e815c719023..820bb591a0d 100644
--- a/openmp/runtime/src/kmp_itt.cpp
+++ b/openmp/runtime/src/kmp_itt.cpp
@@ -22,6 +22,9 @@
 
 #if USE_ITT_NOTIFY
 
+#include "ittnotify_config.h"
+__itt_global __kmp_ittapi_clean_global;
+extern __itt_global __kmp_itt__ittapi_global;
 kmp_int32 __kmp_barrier_domain_count;
 kmp_int32 __kmp_region_domain_count;
 __itt_domain *__kmp_itt_barrier_domains[KMP_MAX_FRAME_DOMAINS];
@@ -53,6 +56,12 @@ kmp_bootstrap_lock_t __kmp_itt_debug_lock =
 
 #endif // USE_ITT_NOTIFY
 
+void __kmp_itt_reset() {
+#if USE_ITT_NOTIFY
+  __kmp_itt__ittapi_global = __kmp_ittapi_clean_global;
+#endif
+}
+
 void __kmp_itt_initialize() {
 
 // ITTNotify library is loaded and initialized at first call to any ittnotify
@@ -60,6 +69,9 @@ void __kmp_itt_initialize() {
 // RTL version to ITTNotify.
 
 #if USE_ITT_NOTIFY
+  // Backup a clean global state
+  __kmp_ittapi_clean_global = __kmp_itt__ittapi_global;
+
   // Report OpenMP RTL version.
   kmp_str_buf_t buf;
   __itt_mark_type version;
diff --git a/openmp/runtime/src/kmp_itt.h b/openmp/runtime/src/kmp_itt.h
index e96119476a2..2062e01c4b1 100644
--- a/openmp/runtime/src/kmp_itt.h
+++ b/openmp/runtime/src/kmp_itt.h
@@ -42,6 +42,7 @@ extern void __kmp_itt_fini_ittlib(void);
 
 void __kmp_itt_initialize();
 void __kmp_itt_destroy();
+void __kmp_itt_reset();
 
 // -----------------------------------------------------------------------------
 // New stuff for reporting high-level constructs.
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 8035e077b39..04a9ef3eea6 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -94,7 +94,7 @@ static int __kmp_unregister_root_other_thread(int gtid);
 #endif
 static void __kmp_unregister_library(void); // called by __kmp_internal_end()
 static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
-static kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
+kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
 
 /* Calculate the identifier of the current thread */
 /* fast (and somewhat portable) way to get unique identifier of executing
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 03511ba8d96..97baaf701ed 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -2727,7 +2727,7 @@ static void __kmp_enable_tasking(kmp_task_team_t *task_team,
 static kmp_task_team_t *__kmp_free_task_teams =
     NULL; // Free list for task_team data structures
 // Lock for task team data structures
-static kmp_bootstrap_lock_t __kmp_task_team_lock =
+kmp_bootstrap_lock_t __kmp_task_team_lock =
     KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
 
 // __kmp_alloc_task_deque:
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index fa9b1c53888..d7e2ed87875 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -1253,16 +1253,22 @@ void __kmp_disable(int *old_state) {
 #endif
 }
 
-static void __kmp_atfork_prepare(void) { /*  nothing to do  */
+static void __kmp_atfork_prepare(void) {
+  __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
 }
 
-static void __kmp_atfork_parent(void) { /*  nothing to do  */
+static void __kmp_atfork_parent(void) {
+  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
 }
 
 /* Reset the library so execution in the child starts "all over again" with
    clean data structures in initial states.  Don't worry about freeing memory
    allocated by parent, just abandon it to be safe. */
 static void __kmp_atfork_child(void) {
+  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
   /* TODO make sure this is done right for nested/sibling */
   // ATT:  Memory leaks are here? TODO: Check it and fix.
   /* KMP_ASSERT( 0 ); */
@@ -1307,6 +1313,10 @@ static void __kmp_atfork_child(void) {
   __kmp_all_nth = 0;
   TCW_4(__kmp_nth, 0);
 
+  __kmp_thread_pool = NULL;
+  __kmp_thread_pool_insert_pt = NULL;
+  __kmp_team_pool = NULL;
+
   /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
      here so threadprivate doesn't use stale data */
   KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
@@ -1329,6 +1339,9 @@ static void __kmp_atfork_child(void) {
   __kmp_init_bootstrap_lock(&__kmp_initz_lock);
   __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
   __kmp_init_bootstrap_lock(&__kmp_console_lock);
+  __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
+
+  __kmp_itt_reset(); // reset ITT's global state
 
   /* This is necessary to make sure no stale data is left around */
   /* AC: customers complain that we use unsafe routines in the atfork