Enable yield cycle on Linux

This change allows the runtime to turn __kmp_yield() on/off repeatedly on Linux.
This feature was removed when disabling monitor thread, but there are
applications that perform better with this feature on.

Patch by Hansang Bae

Differential Revision: https://reviews.llvm.org/D29227

llvm-svn: 295203

4 files changed, 14 insertions, 11 deletions

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 4eb586c2f4f..da29338578a 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -894,15 +894,19 @@ extern int __kmp_place_num_threads_per_core;
    // HW TSC is used to reduce overhead (clock tick instead of nanosecond).
    extern double __kmp_ticks_per_nsec;
 #  define KMP_NOW() __kmp_hardware_timestamp()
+#  define KMP_NOW_MSEC() ((kmp_uint64)(KMP_NOW()/__kmp_ticks_per_nsec)/KMP_USEC_PER_SEC)
 #  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC * __kmp_ticks_per_nsec)
 #  define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
 # else
    // System time is retrieved sporadically while blocking.
    extern kmp_uint64 __kmp_now_nsec();
 #  define KMP_NOW() __kmp_now_nsec()
+#  define KMP_NOW_MSEC() (KMP_NOW()/KMP_USEC_PER_SEC)
 #  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC)
 #  define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
 # endif
+# define KMP_YIELD_NOW() (KMP_NOW_MSEC() / KMP_MAX(__kmp_dflt_blocktime, 1)  \
+                         % (__kmp_yield_on_count + __kmp_yield_off_count) < (kmp_uint32)__kmp_yield_on_count)
 #endif // KMP_USE_MONITOR
 
 #define KMP_MIN_STATSCOLS       40
@@ -2674,10 +2678,10 @@ extern kmp_uint32 __kmp_yield_next;
 
 #if KMP_USE_MONITOR
 extern kmp_uint32 __kmp_yielding_on;
+#endif
 extern kmp_uint32 __kmp_yield_cycle;
 extern kmp_int32  __kmp_yield_on_count;
 extern kmp_int32  __kmp_yield_off_count;
-#endif
 
 /* ------------------------------------------------------------------------- */
 extern int        __kmp_allThreadsSpecified;
diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp
index 5e73f87c31e..5c8065733fa 100644
--- a/openmp/runtime/src/kmp_global.cpp
+++ b/openmp/runtime/src/kmp_global.cpp
@@ -354,6 +354,7 @@ kmp_uint32 __kmp_yield_next = KMP_NEXT_WAIT;
 
 #if KMP_USE_MONITOR
 kmp_uint32 __kmp_yielding_on = 1;
+#endif
 #if KMP_OS_CNK
 kmp_uint32 __kmp_yield_cycle = 0;
 #else
@@ -361,7 +362,6 @@ kmp_uint32 __kmp_yield_cycle = 1;     /* Yield-cycle is on by default */
 #endif
 kmp_int32  __kmp_yield_on_count = 10; /* By default, yielding is on for 10 monitor periods. */
 kmp_int32  __kmp_yield_off_count = 1; /* By default, yielding is off for 1 monitor periods. */
-#endif
 /* ----------------------------------------------------- */
 
 
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 9dd86512048..3f8ec39a5dc 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -3798,7 +3798,6 @@ __kmp_stg_print_par_range_env( kmp_str_buf_t * buffer, char const * name, void *
     }
 } // __kmp_stg_print_par_range_env
 
-#if KMP_USE_MONITOR
 // -------------------------------------------------------------------------------------------------
 // KMP_YIELD_CYCLE, KMP_YIELD_ON, KMP_YIELD_OFF
 // -------------------------------------------------------------------------------------------------
@@ -3834,7 +3833,6 @@ static void
 __kmp_stg_print_yield_off( kmp_str_buf_t * buffer, char const * name, void * data ) {
     __kmp_stg_print_int( buffer, name, __kmp_yield_off_count );
 } // __kmp_stg_print_yield_off
-#endif // KMP_USE_MONITOR
 
 #endif
 
@@ -4740,11 +4738,9 @@ static kmp_setting_t __kmp_stg_table[] = {
     { "KMP_DIAG",                          __kmp_stg_parse_diag,               __kmp_stg_print_diag,               NULL, 0, 0 },
 
     { "KMP_PAR_RANGE",                     __kmp_stg_parse_par_range_env,      __kmp_stg_print_par_range_env,      NULL, 0, 0 },
-#if KMP_USE_MONITOR
     { "KMP_YIELD_CYCLE",                   __kmp_stg_parse_yield_cycle,        __kmp_stg_print_yield_cycle,        NULL, 0, 0 },
     { "KMP_YIELD_ON",                      __kmp_stg_parse_yield_on,           __kmp_stg_print_yield_on,           NULL, 0, 0 },
     { "KMP_YIELD_OFF",                     __kmp_stg_parse_yield_off,          __kmp_stg_print_yield_off,          NULL, 0, 0 },
-#endif
 #endif // KMP_DEBUG
 
     { "KMP_ALIGN_ALLOC",                   __kmp_stg_parse_align_alloc,        __kmp_stg_print_align_alloc,        NULL, 0, 0 },
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index b9490ea65d3..8cba1ff5607 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -1812,13 +1812,16 @@ __kmp_resume_monitor()
 void
 __kmp_yield( int cond )
 {
-    if (cond
+    if (!cond)
+        return;
 #if KMP_USE_MONITOR
-        && __kmp_yielding_on
+    if (!__kmp_yielding_on)
+        return;
+#else
+    if (__kmp_yield_cycle && !KMP_YIELD_NOW())
+        return;
 #endif
-    ) {
-        sched_yield();
-    }
+    sched_yield();
 }
 
 /* ------------------------------------------------------------------------ */