Fix some bugs in OMPT support

1.) in kmp_csupport.c, move computation of parameters only needed for OMPT tracing
inside a conditional to reduce overhead if not receiving ompt_event_master_begin
callbacks.
2.) in kmp_gsupport.c, remove spurious reset of OMPT reenter_runtime_frame (which 
is set in its caller, GOMP_parallel_start correct placement of #if OMP_TRACE so 
that state is maintained even if tracing support not included.  
3.) in z_Linux_util.c, add architecture independent support for OMPT by setting 
and resetting OMPT's exit_frame_ptr before and after invoking a microtask.  
4.) On the Intel MIC, the loader refuses to retain static symbols in the 
libomp.so shared library, even though tools need them. The loader could not be
bullied into doing so. To accommodate this, I changed the visibility of OMPT 
placeholder functions to public. This required additions in exports.so.txt, 
adding extern "C" scoping in ompt-general.c so that the public placeholder
symbols won't be mangled.

Patch by John Mellor-Crummey

Differential Revision: http://reviews.llvm.org/D11062

llvm-svn: 242052

8 files changed, 80 insertions, 35 deletions

diff --git a/openmp/runtime/src/exports_so.txt b/openmp/runtime/src/exports_so.txt
index 06ef2e708e2..d7ade4414ba 100644
--- a/openmp/runtime/src/exports_so.txt
+++ b/openmp/runtime/src/exports_so.txt
@@ -23,6 +23,16 @@ VERSION {
         omp_*;     # Standard OpenMP functions.
         ompt_initialize;     # OMPT initialization interface
         ompt_control;        # OMPT control interface
+        
+        #
+        # OMPT state placeholders
+        #
+        ompt_idle;
+        ompt_overhead;
+        ompt_barrier_wait;
+        ompt_task_wait;
+        ompt_mutex_wait;
+
         ompc_*;    # omp.h renames some standard functions to ompc_*.
         kmp_*;     # Intel extensions.
         kmpc_*;    # Intel extensions.
diff --git a/openmp/runtime/src/include/30/ompt.h.var b/openmp/runtime/src/include/30/ompt.h.var
index 013f8d8c87a..b071c75b76d 100644
--- a/openmp/runtime/src/include/30/ompt.h.var
+++ b/openmp/runtime/src/include/30/ompt.h.var
@@ -34,11 +34,11 @@
     macro (ompt_get_thread_id)
 
 #define FOREACH_OMPT_PLACEHOLDER_FN(macro)  \
-    macro (omp_idle)                        \
-    macro (omp_overhead)                    \
-    macro (omp_barrier_wait)                \
-    macro (omp_task_wait)                   \
-    macro (omp_mutex_wait)
+    macro (ompt_idle)                       \
+    macro (ompt_overhead)                   \
+    macro (ompt_barrier_wait)               \
+    macro (ompt_task_wait)                  \
+    macro (ompt_mutex_wait)
 
 #define FOREACH_OMPT_STATE(macro)                                                               \
                                                                                                 \
diff --git a/openmp/runtime/src/include/40/ompt.h.var b/openmp/runtime/src/include/40/ompt.h.var
index 851cc680371..1414dfb78d1 100644
--- a/openmp/runtime/src/include/40/ompt.h.var
+++ b/openmp/runtime/src/include/40/ompt.h.var
@@ -34,11 +34,11 @@
     macro (ompt_get_thread_id)
 
 #define FOREACH_OMPT_PLACEHOLDER_FN(macro)  \
-    macro (omp_idle)                        \
-    macro (omp_overhead)                    \
-    macro (omp_barrier_wait)                \
-    macro (omp_task_wait)                   \
-    macro (omp_mutex_wait)
+    macro (ompt_idle)                       \
+    macro (ompt_overhead)                   \
+    macro (ompt_barrier_wait)               \
+    macro (ompt_task_wait)                  \
+    macro (ompt_mutex_wait)
 
 #define FOREACH_OMPT_STATE(macro)                                                               \
                                                                                                 \
diff --git a/openmp/runtime/src/include/41/ompt.h.var b/openmp/runtime/src/include/41/ompt.h.var
index d0011acabf8..acc6d052121 100644
--- a/openmp/runtime/src/include/41/ompt.h.var
+++ b/openmp/runtime/src/include/41/ompt.h.var
@@ -34,11 +34,11 @@
     macro (ompt_get_thread_id)
 
 #define FOREACH_OMPT_PLACEHOLDER_FN(macro)  \
-    macro (omp_idle)                        \
-    macro (omp_overhead)                    \
-    macro (omp_barrier_wait)                \
-    macro (omp_task_wait)                   \
-    macro (omp_mutex_wait)
+    macro (ompt_idle)                       \
+    macro (ompt_overhead)                   \
+    macro (ompt_barrier_wait)               \
+    macro (ompt_task_wait)                  \
+    macro (ompt_mutex_wait)
 
 #define FOREACH_OMPT_STATE(macro)                                                               \
                                                                                                 \
diff --git a/openmp/runtime/src/kmp_csupport.c b/openmp/runtime/src/kmp_csupport.c
index fae12c5fd53..70fe3ddc7a3 100644
--- a/openmp/runtime/src/kmp_csupport.c
+++ b/openmp/runtime/src/kmp_csupport.c
@@ -701,10 +701,11 @@ __kmpc_master(ident_t *loc, kmp_int32 global_tid)
 
 #if OMPT_SUPPORT && OMPT_TRACE
     if (status) {
-        kmp_info_t  *this_thr        = __kmp_threads[ global_tid ];
-        kmp_team_t  *team            = this_thr -> th.th_team;
         if ((ompt_status == ompt_status_track_callback) &&
             ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
+            kmp_info_t  *this_thr        = __kmp_threads[ global_tid ];
+            kmp_team_t  *team            = this_thr -> th.th_team;
+
             int  tid = __kmp_tid_from_gtid( global_tid );
             ompt_callbacks.ompt_callback(ompt_event_master_begin)(
                 team->t.ompt_team_info.parallel_id,
diff --git a/openmp/runtime/src/kmp_gsupport.c b/openmp/runtime/src/kmp_gsupport.c
index 3e4d996a2a5..9397e6fd988 100644
--- a/openmp/runtime/src/kmp_gsupport.c
+++ b/openmp/runtime/src/kmp_gsupport.c
@@ -349,11 +349,6 @@ __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), mic
     va_list ap;
     va_start(ap, argc);
 
-#if OMPT_SUPPORT
-    team->t.t_implicit_task_taskdata[tid].
-        ompt_task_info.frame.reenter_runtime_frame = NULL;
-#endif
-
     rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc,
 #if OMPT_SUPPORT
       VOLATILE_CAST(void *) unwrapped_task,
@@ -372,8 +367,9 @@ __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), mic
         __kmp_run_before_invoked_task(gtid, tid, thr, team);
     }
 
-#if OMPT_SUPPORT && OMPT_TRACE
+#if OMPT_SUPPORT 
     if (ompt_status & ompt_status_track) {
+#if OMPT_TRACE
         ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
         ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
 
@@ -383,6 +379,7 @@ __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), mic
             ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
                 team_info->parallel_id, task_info->task_id);
         }
+#endif
         thr->th.ompt_thread_info.state = ompt_state_work_parallel;
     }
 #endif
diff --git a/openmp/runtime/src/ompt-general.c b/openmp/runtime/src/ompt-general.c
index 327de8db60e..2e5d4ff06fb 100644
--- a/openmp/runtime/src/ompt-general.c
+++ b/openmp/runtime/src/ompt-general.c
@@ -299,49 +299,62 @@ OMPT_API_ROUTINE void *ompt_get_task_function(int depth)
  * placeholders
  ****************************************************************************/
 
+// Don't define this as static. The loader may choose to eliminate the symbol
+// even though it is needed by tools.  
+#define OMPT_API_PLACEHOLDER 
 
-OMPT_API_ROUTINE void omp_idle(void)
+// Ensure that placeholders don't have mangled names in the symbol table.
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+OMPT_API_PLACEHOLDER void ompt_idle(void)  
 {
-    // this function is a placeholder used to represent the calling context of
+    // This function is a placeholder used to represent the calling context of
     // idle OpenMP worker threads. It is not meant to be invoked.
     assert(0);
 }
 
 
-OMPT_API_ROUTINE void omp_overhead(void)
+OMPT_API_PLACEHOLDER void ompt_overhead(void)
 {
-    // this function is a placeholder used to represent the OpenMP context of
+    // This function is a placeholder used to represent the OpenMP context of
     // threads working in the OpenMP runtime.  It is not meant to be invoked.
     assert(0);
 }
 
 
-OMPT_API_ROUTINE void omp_barrier_wait(void)
+OMPT_API_PLACEHOLDER void ompt_barrier_wait(void)
 {
-    // this function is a placeholder used to represent the OpenMP context of
+    // This function is a placeholder used to represent the OpenMP context of
     // threads waiting for a barrier in the OpenMP runtime. It is not meant
     // to be invoked.
     assert(0);
 }
 
 
-OMPT_API_ROUTINE void omp_task_wait(void)
+OMPT_API_PLACEHOLDER void ompt_task_wait(void)
 {
-    // this function is a placeholder used to represent the OpenMP context of
+    // This function is a placeholder used to represent the OpenMP context of
     // threads waiting for a task in the OpenMP runtime. It is not meant
     // to be invoked.
     assert(0);
 }
 
 
-OMPT_API_ROUTINE void omp_mutex_wait(void)
+OMPT_API_PLACEHOLDER void ompt_mutex_wait(void)
 {
-    // this function is a placeholder used to represent the OpenMP context of
+    // This function is a placeholder used to represent the OpenMP context of
     // threads waiting for a mutex in the OpenMP runtime. It is not meant
     // to be invoked.
     assert(0);
 }
 
+#ifdef __cplusplus
+};
+#endif
+
 
 /*****************************************************************************
  * compatability
diff --git a/openmp/runtime/src/z_Linux_util.c b/openmp/runtime/src/z_Linux_util.c
index b3eaf2c59a5..9f34d3460ea 100644
--- a/openmp/runtime/src/z_Linux_util.c
+++ b/openmp/runtime/src/z_Linux_util.c
@@ -2619,7 +2619,11 @@ __kmp_get_load_balance( int max )
 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
 
 int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
-        void *p_argv[] )
+        void *p_argv[] 
+#if OMPT_SUPPORT
+        , void **exit_frame_ptr
+#endif
+)
 {
     int argc_full = argc + 2;
     int i;
@@ -2628,6 +2632,9 @@ int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
     void *args[argc_full];
     void *idp[2];
 
+#if OMPT_SUPPORT
+    *exit_frame_ptr = __builtin_frame_address(0);
+#endif
     /* We're only passing pointers to the target. */
     for (i = 0; i < argc_full; i++)
         types[i] = &ffi_type_pointer;
@@ -2647,6 +2654,10 @@ int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
 
     ffi_call(&cif, (void (*)(void))pkfn, NULL, args);
 
+#if OMPT_SUPPORT
+    *exit_frame_ptr = 0;
+#endif
+
     return 1;
 }
 
@@ -2659,7 +2670,16 @@ int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
 int
 __kmp_invoke_microtask( microtask_t pkfn,
                         int gtid, int tid,
-                        int argc, void *p_argv[] ) {
+                        int argc, void *p_argv[] 
+#if OMPT_SUPPORT
+                        , void **exit_frame_ptr
+#endif
+) 
+{
+#if OMPT_SUPPORT
+  *exit_frame_ptr = __builtin_frame_address(0);
+#endif
+
   switch (argc) {
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
@@ -2729,6 +2749,10 @@ __kmp_invoke_microtask( microtask_t pkfn,
     break;
   }
 
+#if OMPT_SUPPORT
+  *exit_frame_ptr = 0;
+#endif
+
   return 1;
 }