diff options
Diffstat (limited to 'openmp/runtime/src/z_Windows_NT_util.cpp')
| -rw-r--r-- | openmp/runtime/src/z_Windows_NT_util.cpp | 1772 |
1 files changed, 1772 insertions, 0 deletions
diff --git a/openmp/runtime/src/z_Windows_NT_util.cpp b/openmp/runtime/src/z_Windows_NT_util.cpp new file mode 100644 index 00000000000..1a371303e1a --- /dev/null +++ b/openmp/runtime/src/z_Windows_NT_util.cpp @@ -0,0 +1,1772 @@ +/* + * z_Windows_NT_util.c -- platform specific routines. + */ + + +//===----------------------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.txt for details. +// +//===----------------------------------------------------------------------===// + + +#include "kmp.h" +#include "kmp_itt.h" +#include "kmp_i18n.h" +#include "kmp_io.h" +#include "kmp_wait_release.h" +#include "kmp_affinity.h" + +/* This code is related to NtQuerySystemInformation() function. This function + is used in the Load balance algorithm for OMP_DYNAMIC=true to find the + number of running threads in the system. */ + +#include <ntstatus.h> +#include <ntsecapi.h> // UNICODE_STRING + +enum SYSTEM_INFORMATION_CLASS { + SystemProcessInformation = 5 +}; // SYSTEM_INFORMATION_CLASS + +struct CLIENT_ID { + HANDLE UniqueProcess; + HANDLE UniqueThread; +}; // struct CLIENT_ID + +enum THREAD_STATE { + StateInitialized, + StateReady, + StateRunning, + StateStandby, + StateTerminated, + StateWait, + StateTransition, + StateUnknown +}; // enum THREAD_STATE + +struct VM_COUNTERS { + SIZE_T PeakVirtualSize; + SIZE_T VirtualSize; + ULONG PageFaultCount; + SIZE_T PeakWorkingSetSize; + SIZE_T WorkingSetSize; + SIZE_T QuotaPeakPagedPoolUsage; + SIZE_T QuotaPagedPoolUsage; + SIZE_T QuotaPeakNonPagedPoolUsage; + SIZE_T QuotaNonPagedPoolUsage; + SIZE_T PagefileUsage; + SIZE_T PeakPagefileUsage; + SIZE_T PrivatePageCount; +}; // struct VM_COUNTERS + +struct SYSTEM_THREAD { + LARGE_INTEGER KernelTime; + LARGE_INTEGER UserTime; + LARGE_INTEGER CreateTime; + ULONG WaitTime; + LPVOID StartAddress; + CLIENT_ID ClientId; + DWORD Priority; + LONG BasePriority; + ULONG ContextSwitchCount; + THREAD_STATE State; + ULONG WaitReason; +}; // SYSTEM_THREAD + +KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 ); +#if KMP_ARCH_X86 + KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 ); +#else + KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 ); +#endif + +struct SYSTEM_PROCESS_INFORMATION { + ULONG NextEntryOffset; + ULONG NumberOfThreads; + LARGE_INTEGER Reserved[ 3 ]; + LARGE_INTEGER CreateTime; + LARGE_INTEGER UserTime; + LARGE_INTEGER KernelTime; + UNICODE_STRING ImageName; + DWORD BasePriority; + HANDLE ProcessId; + HANDLE ParentProcessId; + ULONG HandleCount; + ULONG Reserved2[ 2 ]; + VM_COUNTERS VMCounters; + IO_COUNTERS IOCounters; + SYSTEM_THREAD Threads[ 1 ]; +}; // SYSTEM_PROCESS_INFORMATION +typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION; + +KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 ); +KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 ); +KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 ); +#if KMP_ARCH_X86 + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 ); +#else + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 ); + KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 ); +#endif + +typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG ); +NtQuerySystemInformation_t NtQuerySystemInformation = NULL; + +HMODULE ntdll = NULL; + +/* End of NtQuerySystemInformation()-related code */ + +static HMODULE kernel32 = NULL; + +/* ----------------------------------------------------------------------------------- */ +/* ----------------------------------------------------------------------------------- */ + +#if KMP_HANDLE_SIGNALS + typedef void (* sig_func_t )( int ); + static sig_func_t __kmp_sighldrs[ NSIG ]; + static int __kmp_siginstalled[ NSIG ]; +#endif + +#if KMP_USE_MONITOR +static HANDLE __kmp_monitor_ev; +#endif +static kmp_int64 __kmp_win32_time; +double __kmp_win32_tick; + +int __kmp_init_runtime = FALSE; +CRITICAL_SECTION __kmp_win32_section; + +void +__kmp_win32_mutex_init( kmp_win32_mutex_t *mx ) +{ + InitializeCriticalSection( & mx->cs ); +#if USE_ITT_BUILD + __kmp_itt_system_object_created( & mx->cs, "Critical Section" ); +#endif /* USE_ITT_BUILD */ +} + +void +__kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx ) +{ + DeleteCriticalSection( & mx->cs ); +} + +void +__kmp_win32_mutex_lock( kmp_win32_mutex_t *mx ) +{ + EnterCriticalSection( & mx->cs ); +} + +void +__kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx ) +{ + LeaveCriticalSection( & mx->cs ); +} + +void +__kmp_win32_cond_init( kmp_win32_cond_t *cv ) +{ + cv->waiters_count_ = 0; + cv->wait_generation_count_ = 0; + cv->release_count_ = 0; + + /* Initialize the critical section */ + __kmp_win32_mutex_init( & cv->waiters_count_lock_ ); + + /* Create a manual-reset event. */ + cv->event_ = CreateEvent( NULL, // no security + TRUE, // manual-reset + FALSE, // non-signaled initially + NULL ); // unnamed +#if USE_ITT_BUILD + __kmp_itt_system_object_created( cv->event_, "Event" ); +#endif /* USE_ITT_BUILD */ +} + +void +__kmp_win32_cond_destroy( kmp_win32_cond_t *cv ) +{ + __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ ); + __kmp_free_handle( cv->event_ ); + memset( cv, '\0', sizeof( *cv ) ); +} + +/* TODO associate cv with a team instead of a thread so as to optimize + * the case where we wake up a whole team */ + +void +__kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load ) +{ + int my_generation; + int last_waiter; + + /* Avoid race conditions */ + __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); + + /* Increment count of waiters */ + cv->waiters_count_++; + + /* Store current generation in our activation record. */ + my_generation = cv->wait_generation_count_; + + __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); + __kmp_win32_mutex_unlock( mx ); + + for (;;) { + int wait_done; + + /* Wait until the event is signaled */ + WaitForSingleObject( cv->event_, INFINITE ); + + __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); + + /* Exit the loop when the <cv->event_> is signaled and + * there are still waiting threads from this <wait_generation> + * that haven't been released from this wait yet. */ + wait_done = ( cv->release_count_ > 0 ) && + ( cv->wait_generation_count_ != my_generation ); + + __kmp_win32_mutex_unlock( &cv->waiters_count_lock_); + + /* there used to be a semicolon after the if statement, + * it looked like a bug, so i removed it */ + if( wait_done ) + break; + } + + __kmp_win32_mutex_lock( mx ); + __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); + + cv->waiters_count_--; + cv->release_count_--; + + last_waiter = ( cv->release_count_ == 0 ); + + __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); + + if( last_waiter ) { + /* We're the last waiter to be notified, so reset the manual event. */ + ResetEvent( cv->event_ ); + } +} + +void +__kmp_win32_cond_broadcast( kmp_win32_cond_t *cv ) +{ + __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); + + if( cv->waiters_count_ > 0 ) { + SetEvent( cv->event_ ); + /* Release all the threads in this generation. */ + + cv->release_count_ = cv->waiters_count_; + + /* Start a new generation. */ + cv->wait_generation_count_++; + } + + __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); +} + +void +__kmp_win32_cond_signal( kmp_win32_cond_t *cv ) +{ + __kmp_win32_cond_broadcast( cv ); +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void +__kmp_enable( int new_state ) +{ + if (__kmp_init_runtime) + LeaveCriticalSection( & __kmp_win32_section ); +} + +void +__kmp_disable( int *old_state ) +{ + *old_state = 0; + + if (__kmp_init_runtime) + EnterCriticalSection( & __kmp_win32_section ); +} + +void +__kmp_suspend_initialize( void ) +{ + /* do nothing */ +} + +static void +__kmp_suspend_initialize_thread( kmp_info_t *th ) +{ + if ( ! TCR_4( th->th.th_suspend_init ) ) { + /* this means we haven't initialized the suspension pthread objects for this thread + in this instance of the process */ + __kmp_win32_cond_init( &th->th.th_suspend_cv ); + __kmp_win32_mutex_init( &th->th.th_suspend_mx ); + TCW_4( th->th.th_suspend_init, TRUE ); + } +} + +void +__kmp_suspend_uninitialize_thread( kmp_info_t *th ) +{ + if ( TCR_4( th->th.th_suspend_init ) ) { + /* this means we have initialize the suspension pthread objects for this thread + in this instance of the process */ + __kmp_win32_cond_destroy( & th->th.th_suspend_cv ); + __kmp_win32_mutex_destroy( & th->th.th_suspend_mx ); + TCW_4( th->th.th_suspend_init, FALSE ); + } +} + +/* This routine puts the calling thread to sleep after setting the + * sleep bit for the indicated flag variable to true. + */ +template <class C> +static inline void __kmp_suspend_template( int th_gtid, C *flag ) +{ + kmp_info_t *th = __kmp_threads[th_gtid]; + int status; + typename C::flag_t old_spin; + + KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) ); + + __kmp_suspend_initialize_thread( th ); + __kmp_win32_mutex_lock( &th->th.th_suspend_mx ); + + KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n", + th_gtid, flag->get() ) ); + + /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread + gets called first? + */ + old_spin = flag->set_sleeping(); + + KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n", + th_gtid, flag->get(), *(flag->get()) ) ); + + if ( flag->done_check_val(old_spin) ) { + old_spin = flag->unset_sleeping(); + KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n", + th_gtid, flag->get()) ); + } else { +#ifdef DEBUG_SUSPEND + __kmp_suspend_count++; +#endif + /* Encapsulate in a loop as the documentation states that this may + * "with low probability" return when the condition variable has + * not been signaled or broadcast + */ + int deactivated = FALSE; + TCW_PTR(th->th.th_sleep_loc, (void *)flag); + while ( flag->is_sleeping() ) { + KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n", + th_gtid ) ); + // Mark the thread as no longer active (only in the first iteration of the loop). + if ( ! deactivated ) { + th->th.th_active = FALSE; + if ( th->th.th_active_in_pool ) { + th->th.th_active_in_pool = FALSE; + KMP_TEST_THEN_DEC32( + (kmp_int32 *) &__kmp_thread_pool_active_nth ); + KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 ); + } + deactivated = TRUE; + + __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 ); + } + else { + __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 ); + } + +#ifdef KMP_DEBUG + if( flag->is_sleeping() ) { + KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid )); + } +#endif /* KMP_DEBUG */ + + } // while + + // Mark the thread as active again (if it was previous marked as inactive) + if ( deactivated ) { + th->th.th_active = TRUE; + if ( TCR_4(th->th.th_in_pool) ) { + KMP_TEST_THEN_INC32( + (kmp_int32 *) &__kmp_thread_pool_active_nth ); + th->th.th_active_in_pool = TRUE; + } + } + } + + __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); + + KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) ); +} + +void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) { + __kmp_suspend_template(th_gtid, flag); +} +void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) { + __kmp_suspend_template(th_gtid, flag); +} +void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { + __kmp_suspend_template(th_gtid, flag); +} + + +/* This routine signals the thread specified by target_gtid to wake up + * after setting the sleep bit indicated by the flag argument to FALSE + */ +template <class C> +static inline void __kmp_resume_template( int target_gtid, C *flag ) +{ + kmp_info_t *th = __kmp_threads[target_gtid]; + int status; + +#ifdef KMP_DEBUG + int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; +#endif + + KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) ); + + __kmp_suspend_initialize_thread( th ); + __kmp_win32_mutex_lock( &th->th.th_suspend_mx ); + + if (!flag) { // coming from __kmp_null_resume_wrapper + flag = (C *)th->th.th_sleep_loc; + } + + // First, check if the flag is null or its type has changed. If so, someone else woke it up. + if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to + KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n", + gtid, target_gtid, NULL ) ); + __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); + return; + } + else { + typename C::flag_t old_spin = flag->unset_sleeping(); + if ( !flag->is_sleeping_val(old_spin) ) { + KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): " + "%u => %u\n", + gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) ); + __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); + return; + } + } + TCW_PTR(th->th.th_sleep_loc, NULL); + + KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n", + gtid, target_gtid, flag->get() ) ); + + __kmp_win32_cond_signal( &th->th.th_suspend_cv ); + __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); + + KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n", + gtid, target_gtid ) ); +} + +void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) { + __kmp_resume_template(target_gtid, flag); +} +void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) { + __kmp_resume_template(target_gtid, flag); +} +void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { + __kmp_resume_template(target_gtid, flag); +} + + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void +__kmp_yield( int cond ) +{ + if (cond) + Sleep(0); +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void +__kmp_gtid_set_specific( int gtid ) +{ + if( __kmp_init_gtid ) { + KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n", + gtid, __kmp_gtid_threadprivate_key )); + if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) ) + KMP_FATAL( TLSSetValueFailed ); + } else { + KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) ); + } +} + +int +__kmp_gtid_get_specific() +{ + int gtid; + if( !__kmp_init_gtid ) { + KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) ); + return KMP_GTID_SHUTDOWN; + } + gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key ); + if ( gtid == 0 ) { + gtid = KMP_GTID_DNE; + } + else { + gtid--; + } + KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", + __kmp_gtid_threadprivate_key, gtid )); + return gtid; +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void +__kmp_affinity_bind_thread( int proc ) +{ + if (__kmp_num_proc_groups > 1) { + // + // Form the GROUP_AFFINITY struct directly, rather than filling + // out a bit vector and calling __kmp_set_system_affinity(). + // + GROUP_AFFINITY ga; + KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups + * CHAR_BIT * sizeof(DWORD_PTR)))); + ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR)); + ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR))); + ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0; + + KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL); + if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) { + DWORD error = GetLastError(); + if (__kmp_affinity_verbose) { // AC: continue silently if not verbose + kmp_msg_t err_code = KMP_ERR( error ); + __kmp_msg( + kmp_ms_warning, + KMP_MSG( CantSetThreadAffMask ), + err_code, + __kmp_msg_null + ); + if (__kmp_generate_warnings == kmp_warnings_off) { + __kmp_str_free(&err_code.str); + } + } + } + } else { + kmp_affin_mask_t *mask; + KMP_CPU_ALLOC_ON_STACK(mask); + KMP_CPU_ZERO(mask); + KMP_CPU_SET(proc, mask); + __kmp_set_system_affinity(mask, TRUE); + KMP_CPU_FREE_FROM_STACK(mask); + } +} + +void +__kmp_affinity_determine_capable( const char *env_var ) +{ + // + // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask(). + // + +#if KMP_GROUP_AFFINITY + KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(DWORD_PTR)); +#else + KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR)); +#endif + + KA_TRACE( 10, ( + "__kmp_affinity_determine_capable: " + "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n", + __kmp_affin_mask_size + ) ); +} + +double +__kmp_read_cpu_time( void ) +{ + FILETIME CreationTime, ExitTime, KernelTime, UserTime; + int status; + double cpu_time; + + cpu_time = 0; + + status = GetProcessTimes( GetCurrentProcess(), &CreationTime, + &ExitTime, &KernelTime, &UserTime ); + + if (status) { + double sec = 0; + + sec += KernelTime.dwHighDateTime; + sec += UserTime.dwHighDateTime; + + /* Shift left by 32 bits */ + sec *= (double) (1 << 16) * (double) (1 << 16); + + sec += KernelTime.dwLowDateTime; + sec += UserTime.dwLowDateTime; + + cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC; + } + + return cpu_time; +} + +int +__kmp_read_system_info( struct kmp_sys_info *info ) +{ + info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */ + info->minflt = 0; /* the number of page faults serviced without any I/O */ + info->majflt = 0; /* the number of page faults serviced that required I/O */ + info->nswap = 0; /* the number of times a process was "swapped" out of memory */ + info->inblock = 0; /* the number of times the file system had to perform input */ + info->oublock = 0; /* the number of times the file system had to perform output */ + info->nvcsw = 0; /* the number of times a context switch was voluntarily */ + info->nivcsw = 0; /* the number of times a context switch was forced */ + + return 1; +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + + +void +__kmp_runtime_initialize( void ) +{ + SYSTEM_INFO info; + kmp_str_buf_t path; + UINT path_size; + + if ( __kmp_init_runtime ) { + return; + }; + +#if KMP_DYNAMIC_LIB + /* Pin dynamic library for the lifetime of application */ + { + // First, turn off error message boxes + UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS); + HMODULE h; + BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS + |GET_MODULE_HANDLE_EX_FLAG_PIN, + (LPCTSTR)&__kmp_serial_initialize, &h); + KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded"); + SetErrorMode (err_mode); // Restore error mode + KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") ); + } +#endif + + InitializeCriticalSection( & __kmp_win32_section ); +#if USE_ITT_BUILD + __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" ); +#endif /* USE_ITT_BUILD */ + __kmp_initialize_system_tick(); + + #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) + if ( ! __kmp_cpuinfo.initialized ) { + __kmp_query_cpuid( & __kmp_cpuinfo ); + }; // if + #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ + + /* Set up minimum number of threads to switch to TLS gtid */ + #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB + // Windows* OS, static library. + /* + New thread may use stack space previously used by another thread, currently terminated. + On Windows* OS, in case of static linking, we do not know the moment of thread termination, + and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead + threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly + finds gtid (by searching through stack addresses of all known threads) for unregistered + foreign tread. + + Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id() + does not search through stacks, but get gtid from TLS immediately. + + --ln + */ + __kmp_tls_gtid_min = 0; + #else + __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; + #endif + + /* for the static library */ + if ( !__kmp_gtid_threadprivate_key ) { + __kmp_gtid_threadprivate_key = TlsAlloc(); + if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) { + KMP_FATAL( TLSOutOfIndexes ); + } + } + + + // + // Load ntdll.dll. + // + /* + Simple + GetModuleHandle( "ntdll.dl" ) + is not suitable due to security issue (see + http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full + path to the library. + */ + __kmp_str_buf_init( & path ); + path_size = GetSystemDirectory( path.str, path.size ); + KMP_DEBUG_ASSERT( path_size > 0 ); + if ( path_size >= path.size ) { + // + // Buffer is too short. Expand the buffer and try again. + // + __kmp_str_buf_reserve( & path, path_size ); + path_size = GetSystemDirectory( path.str, path.size ); + KMP_DEBUG_ASSERT( path_size > 0 ); + }; // if + if ( path_size > 0 && path_size < path.size ) { + // + // Now we have system directory name in the buffer. + // Append backslash and name of dll to form full path, + // + path.used = path_size; + __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" ); + + // + // Now load ntdll using full path. + // + ntdll = GetModuleHandle( path.str ); + } + + KMP_DEBUG_ASSERT( ntdll != NULL ); + if ( ntdll != NULL ) { + NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" ); + } + KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL ); + +#if KMP_GROUP_AFFINITY + // + // Load kernel32.dll. + // Same caveat - must use full system path name. + // + if ( path_size > 0 && path_size < path.size ) { + // + // Truncate the buffer back to just the system path length, + // discarding "\\ntdll.dll", and replacing it with "kernel32.dll". + // + path.used = path_size; + __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" ); + + // + // Load kernel32.dll using full path. + // + kernel32 = GetModuleHandle( path.str ); + KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) ); + + // + // Load the function pointers to kernel32.dll routines + // that may or may not exist on this system. + // + if ( kernel32 != NULL ) { + __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" ); + __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" ); + __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" ); + __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" ); + + KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) ); + KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) ); + KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) ); + KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) ); + KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) ); + + // + // See if group affinity is supported on this system. + // If so, calculate the #groups and #procs. + // + // Group affinity was introduced with Windows* 7 OS and + // Windows* Server 2008 R2 OS. + // + if ( ( __kmp_GetActiveProcessorCount != NULL ) + && ( __kmp_GetActiveProcessorGroupCount != NULL ) + && ( __kmp_GetThreadGroupAffinity != NULL ) + && ( __kmp_SetThreadGroupAffinity != NULL ) + && ( ( __kmp_num_proc_groups + = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) { + // + // Calculate the total number of active OS procs. + // + int i; + + KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) ); + + __kmp_xproc = 0; + + for ( i = 0; i < __kmp_num_proc_groups; i++ ) { + DWORD size = __kmp_GetActiveProcessorCount( i ); + __kmp_xproc += size; + KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) ); + } + } + else { + KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) ); + } + } + } + if ( __kmp_num_proc_groups <= 1 ) { + GetSystemInfo( & info ); + __kmp_xproc = info.dwNumberOfProcessors; + } +#else + GetSystemInfo( & info ); + __kmp_xproc = info.dwNumberOfProcessors; +#endif /* KMP_GROUP_AFFINITY */ + + // + // If the OS said there were 0 procs, take a guess and use a value of 2. + // This is done for Linux* OS, also. Do we need error / warning? + // + if ( __kmp_xproc <= 0 ) { + __kmp_xproc = 2; + } + + KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) ); + + __kmp_str_buf_free( & path ); + +#if USE_ITT_BUILD + __kmp_itt_initialize(); +#endif /* USE_ITT_BUILD */ + + __kmp_init_runtime = TRUE; +} // __kmp_runtime_initialize + +void +__kmp_runtime_destroy( void ) +{ + if ( ! __kmp_init_runtime ) { + return; + } + +#if USE_ITT_BUILD + __kmp_itt_destroy(); +#endif /* USE_ITT_BUILD */ + + /* we can't DeleteCriticalsection( & __kmp_win32_section ); */ + /* due to the KX_TRACE() commands */ + KA_TRACE( 40, ("__kmp_runtime_destroy\n" )); + + if( __kmp_gtid_threadprivate_key ) { + TlsFree( __kmp_gtid_threadprivate_key ); + __kmp_gtid_threadprivate_key = 0; + } + + __kmp_affinity_uninitialize(); + DeleteCriticalSection( & __kmp_win32_section ); + + ntdll = NULL; + NtQuerySystemInformation = NULL; + +#if KMP_ARCH_X86_64 + kernel32 = NULL; + __kmp_GetActiveProcessorCount = NULL; + __kmp_GetActiveProcessorGroupCount = NULL; + __kmp_GetThreadGroupAffinity = NULL; + __kmp_SetThreadGroupAffinity = NULL; +#endif // KMP_ARCH_X86_64 + + __kmp_init_runtime = FALSE; +} + + +void +__kmp_terminate_thread( int gtid ) +{ + kmp_info_t *th = __kmp_threads[ gtid ]; + + if( !th ) return; + + KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) ); + + if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) { + /* It's OK, the thread may have exited already */ + } + __kmp_free_handle( th->th.th_info.ds.ds_thread ); +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void +__kmp_clear_system_time( void ) +{ + BOOL status; + LARGE_INTEGER time; + status = QueryPerformanceCounter( & time ); + __kmp_win32_time = (kmp_int64) time.QuadPart; +} + +void +__kmp_initialize_system_tick( void ) +{ + { + BOOL status; + LARGE_INTEGER freq; + + status = QueryPerformanceFrequency( & freq ); + if (! status) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ), + KMP_ERR( error ), + __kmp_msg_null + ); + + } + else { + __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart; + } + } +} + +/* Calculate the elapsed wall clock time for the user */ + +void +__kmp_elapsed( double *t ) +{ + BOOL status; + LARGE_INTEGER now; + status = QueryPerformanceCounter( & now ); + *t = ((double) now.QuadPart) * __kmp_win32_tick; +} + +/* Calculate the elapsed wall clock tick for the user */ + +void +__kmp_elapsed_tick( double *t ) +{ + *t = __kmp_win32_tick; +} + +void +__kmp_read_system_time( double *delta ) +{ + if (delta != NULL) { + BOOL status; + LARGE_INTEGER now; + + status = QueryPerformanceCounter( & now ); + + *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time)) + * __kmp_win32_tick; + } +} + +/* Return the current time stamp in nsec */ +kmp_uint64 +__kmp_now_nsec() +{ + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + return 1e9 * __kmp_win32_tick * now.QuadPart; +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +void * __stdcall +__kmp_launch_worker( void *arg ) +{ + volatile void *stack_data; + void *exit_val; + void *padding = 0; + kmp_info_t *this_thr = (kmp_info_t *) arg; + int gtid; + + gtid = this_thr->th.th_info.ds.ds_gtid; + __kmp_gtid_set_specific( gtid ); +#ifdef KMP_TDATA_GTID + #error "This define causes problems with LoadLibrary() + declspec(thread) " \ + "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ + "reference: http://support.microsoft.com/kb/118816" + //__kmp_gtid = gtid; +#endif + +#if USE_ITT_BUILD + __kmp_itt_thread_name( gtid ); +#endif /* USE_ITT_BUILD */ + + __kmp_affinity_set_init_mask( gtid, FALSE ); + +#if KMP_ARCH_X86 || KMP_ARCH_X86_64 + // + // Set the FP control regs to be a copy of + // the parallel initialization thread's. + // + __kmp_clear_x87_fpu_status_word(); + __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word ); + __kmp_load_mxcsr( &__kmp_init_mxcsr ); +#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ + + if ( __kmp_stkoffset > 0 && gtid > 0 ) { + padding = KMP_ALLOCA( gtid * __kmp_stkoffset ); + } + + KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive ); + this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId(); + TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE ); + + if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid + TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data); + KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE ); + __kmp_check_stack_overlap( this_thr ); + } + KMP_MB(); + exit_val = __kmp_launch_thread( this_thr ); + KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive ); + TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE ); + KMP_MB(); + return exit_val; +} + +#if KMP_USE_MONITOR +/* The monitor thread controls all of the threads in the complex */ + +void * __stdcall +__kmp_launch_monitor( void *arg ) +{ + DWORD wait_status; + kmp_thread_t monitor; + int status; + int interval; + kmp_info_t *this_thr = (kmp_info_t *) arg; + + KMP_DEBUG_ASSERT(__kmp_init_monitor); + TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started + // TODO: hide "2" in enum (like {true,false,started}) + this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId(); + TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE ); + + KMP_MB(); /* Flush all pending memory write invalidates. */ + KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) ); + + monitor = GetCurrentThread(); + + /* set thread priority */ + status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST ); + if (! status) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantSetThreadPriority ), + KMP_ERR( error ), + __kmp_msg_null + ); + } + + /* register us as monitor */ + __kmp_gtid_set_specific( KMP_GTID_MONITOR ); +#ifdef KMP_TDATA_GTID + #error "This define causes problems with LoadLibrary() + declspec(thread) " \ + "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ + "reference: http://support.microsoft.com/kb/118816" + //__kmp_gtid = KMP_GTID_MONITOR; +#endif + +#if USE_ITT_BUILD + __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread. +#endif /* USE_ITT_BUILD */ + + KMP_MB(); /* Flush all pending memory write invalidates. */ + + interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */ + + while (! TCR_4(__kmp_global.g.g_done)) { + /* This thread monitors the state of the system */ + + KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) ); + + wait_status = WaitForSingleObject( __kmp_monitor_ev, interval ); + + if (wait_status == WAIT_TIMEOUT) { + TCW_4( __kmp_global.g.g_time.dt.t_value, + TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 ); + } + + KMP_MB(); /* Flush all pending memory write invalidates. */ + } + + KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) ); + + status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL ); + if (! status) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantSetThreadPriority ), + KMP_ERR( error ), + __kmp_msg_null + ); + } + + if (__kmp_global.g.g_abort != 0) { + /* now we need to terminate the worker threads */ + /* the value of t_abort is the signal we caught */ + + int gtid; + + KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) ); + + /* terminate the OpenMP worker threads */ + /* TODO this is not valid for sibling threads!! + * the uber master might not be 0 anymore.. */ + for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) + __kmp_terminate_thread( gtid ); + + __kmp_cleanup(); + + Sleep( 0 ); + + KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) ); + + if (__kmp_global.g.g_abort > 0) { + raise( __kmp_global.g.g_abort ); + } + } + + TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE ); + + KMP_MB(); + return arg; +} +#endif + +void +__kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size ) +{ + kmp_thread_t handle; + DWORD idThread; + + KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) ); + + th->th.th_info.ds.ds_gtid = gtid; + + if ( KMP_UBER_GTID(gtid) ) { + int stack_data; + + /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use. + Is it appropriate to just use GetCurrentThread? When should we close this handle? When + unregistering the root? + */ + { + BOOL rc; + rc = DuplicateHandle( + GetCurrentProcess(), + GetCurrentThread(), + GetCurrentProcess(), + &th->th.th_info.ds.ds_thread, + 0, + FALSE, + DUPLICATE_SAME_ACCESS + ); + KMP_ASSERT( rc ); + KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n", + (LPVOID)th, + th->th.th_info.ds.ds_thread ) ); + th->th.th_info.ds.ds_thread_id = GetCurrentThreadId(); + } + if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid + /* we will dynamically update the stack range if gtid_mode == 1 */ + TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); + TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); + TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); + __kmp_check_stack_overlap( th ); + } + } + else { + KMP_MB(); /* Flush all pending memory write invalidates. */ + + /* Set stack size for this thread now. */ + KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC + " bytes\n", stack_size ) ); + + stack_size += gtid * __kmp_stkoffset; + + TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size); + TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); + + KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %" + KMP_SIZE_T_SPEC + " bytes, &__kmp_launch_worker = %p, th = %p, " + "&idThread = %p\n", + (SIZE_T) stack_size, + (LPTHREAD_START_ROUTINE) & __kmp_launch_worker, + (LPVOID) th, &idThread ) ); + + handle = CreateThread( NULL, (SIZE_T) stack_size, + (LPTHREAD_START_ROUTINE) __kmp_launch_worker, + (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread ); + + KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %" + KMP_SIZE_T_SPEC + " bytes, &__kmp_launch_worker = %p, th = %p, " + "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n", + (SIZE_T) stack_size, + (LPTHREAD_START_ROUTINE) & __kmp_launch_worker, + (LPVOID) th, idThread, handle ) ); + + if ( handle == 0 ) { + DWORD error = GetLastError(); + __kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null); + } else { + th->th.th_info.ds.ds_thread = handle; + } + + KMP_MB(); /* Flush all pending memory write invalidates. */ + } + + KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) ); +} + +int +__kmp_still_running(kmp_info_t *th) { + return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0)); +} + +#if KMP_USE_MONITOR +void +__kmp_create_monitor( kmp_info_t *th ) +{ + kmp_thread_t handle; + DWORD idThread; + int ideal, new_ideal; + + if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) { + // We don't need monitor thread in case of MAX_BLOCKTIME + KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) ); + th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op + th->th.th_info.ds.ds_gtid = 0; + TCW_4( __kmp_init_monitor, 2 ); // Signal to stop waiting for monitor creation + return; + } + KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) ); + + KMP_MB(); /* Flush all pending memory write invalidates. */ + + __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL ); + if ( __kmp_monitor_ev == NULL ) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantCreateEvent ), + KMP_ERR( error ), + __kmp_msg_null + ); + }; // if +#if USE_ITT_BUILD + __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" ); +#endif /* USE_ITT_BUILD */ + + th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; + th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; + + // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how + // to automatically expand stacksize based on CreateThread error code. + if ( __kmp_monitor_stksize == 0 ) { + __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; + } + if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) { + __kmp_monitor_stksize = __kmp_sys_min_stksize; + } + + KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n", + (int) __kmp_monitor_stksize ) ); + + TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); + + handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize, + (LPTHREAD_START_ROUTINE) __kmp_launch_monitor, + (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread ); + if (handle == 0) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantCreateThread ), + KMP_ERR( error ), + __kmp_msg_null + ); + } + else + th->th.th_info.ds.ds_thread = handle; + + KMP_MB(); /* Flush all pending memory write invalidates. */ + + KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n", + (void *) th->th.th_info.ds.ds_thread ) ); +} +#endif + +/* + Check to see if thread is still alive. + + NOTE: The ExitProcess(code) system call causes all threads to Terminate + with a exit_val = code. Because of this we can not rely on + exit_val having any particular value. So this routine may + return STILL_ALIVE in exit_val even after the thread is dead. +*/ + +int +__kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val ) +{ + DWORD rc; + rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val ); + if ( rc == 0 ) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( FunctionError, "GetExitCodeThread()" ), + KMP_ERR( error ), + __kmp_msg_null + ); + }; // if + return ( *exit_val == STILL_ACTIVE ); +} + + +void +__kmp_exit_thread( + int exit_status +) { + ExitThread( exit_status ); +} // __kmp_exit_thread + +/* + This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor(). +*/ +static void +__kmp_reap_common( kmp_info_t * th ) +{ + DWORD exit_val; + + KMP_MB(); /* Flush all pending memory write invalidates. */ + + KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) ); + + /* + 2006-10-19: + + There are two opposite situations: + + 1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread + function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very + slow".) + 2. Windows* OS may kill thread before it resets ds_alive flag. + + Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting. + + */ + + { + // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to + // cover this usage also. + void * obj = NULL; + register kmp_uint32 spins; +#if USE_ITT_BUILD + KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive ); +#endif /* USE_ITT_BUILD */ + KMP_INIT_YIELD( spins ); + do { +#if USE_ITT_BUILD + KMP_FSYNC_SPIN_PREPARE( obj ); +#endif /* USE_ITT_BUILD */ + __kmp_is_thread_alive( th, &exit_val ); + KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc ); + KMP_YIELD_SPIN( spins ); + } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) ); +#if USE_ITT_BUILD + if ( exit_val == STILL_ACTIVE ) { + KMP_FSYNC_CANCEL( obj ); + } else { + KMP_FSYNC_SPIN_ACQUIRED( obj ); + }; // if +#endif /* USE_ITT_BUILD */ + } + + __kmp_free_handle( th->th.th_info.ds.ds_thread ); + + /* + * NOTE: The ExitProcess(code) system call causes all threads to Terminate + * with a exit_val = code. Because of this we can not rely on + * exit_val having any particular value. + */ + if ( exit_val == STILL_ACTIVE ) { + KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) ); + } else if ( (void *) exit_val != (void *) th) { + KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) ); + }; // if + + KA_TRACE( 10, + ( + "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n", + th->th.th_info.ds.ds_gtid, + th->th.th_info.ds.ds_thread + ) + ); + + th->th.th_info.ds.ds_thread = 0; + th->th.th_info.ds.ds_tid = KMP_GTID_DNE; + th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; + th->th.th_info.ds.ds_thread_id = 0; + + KMP_MB(); /* Flush all pending memory write invalidates. */ +} + +#if KMP_USE_MONITOR +void +__kmp_reap_monitor( kmp_info_t *th ) +{ + int status; + + KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n", + (void *) th->th.th_info.ds.ds_thread ) ); + + // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. + // If both tid and gtid are 0, it means the monitor did not ever start. + // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. + KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid ); + if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) { + KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") ); + return; + }; // if + + KMP_MB(); /* Flush all pending memory write invalidates. */ + + status = SetEvent( __kmp_monitor_ev ); + if ( status == FALSE ) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantSetEvent ), + KMP_ERR( error ), + __kmp_msg_null + ); + } + KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) ); + __kmp_reap_common( th ); + + __kmp_free_handle( __kmp_monitor_ev ); + + KMP_MB(); /* Flush all pending memory write invalidates. */ +} +#endif + +void +__kmp_reap_worker( kmp_info_t * th ) +{ + KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) ); + __kmp_reap_common( th ); +} + +/* ------------------------------------------------------------------------ */ +/* ------------------------------------------------------------------------ */ + +#if KMP_HANDLE_SIGNALS + + +static void +__kmp_team_handler( int signo ) +{ + if ( __kmp_global.g.g_abort == 0 ) { + // Stage 1 signal handler, let's shut down all of the threads. + if ( __kmp_debug_buf ) { + __kmp_dump_debug_buffer(); + }; // if + KMP_MB(); // Flush all pending memory write invalidates. + TCW_4( __kmp_global.g.g_abort, signo ); + KMP_MB(); // Flush all pending memory write invalidates. + TCW_4( __kmp_global.g.g_done, TRUE ); + KMP_MB(); // Flush all pending memory write invalidates. + } +} // __kmp_team_handler + + + +static +sig_func_t __kmp_signal( int signum, sig_func_t handler ) { + sig_func_t old = signal( signum, handler ); + if ( old == SIG_ERR ) { + int error = errno; + __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null ); + }; // if + return old; +} + +static void +__kmp_install_one_handler( + int sig, + sig_func_t handler, + int parallel_init +) { + sig_func_t old; + KMP_MB(); /* Flush all pending memory write invalidates. */ + KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) ); + if ( parallel_init ) { + old = __kmp_signal( sig, handler ); + // SIG_DFL on Windows* OS in NULL or 0. + if ( old == __kmp_sighldrs[ sig ] ) { + __kmp_siginstalled[ sig ] = 1; + } else { + // Restore/keep user's handler if one previously installed. + old = __kmp_signal( sig, old ); + }; // if + } else { + // Save initial/system signal handlers to see if user handlers installed. + // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with + // parallel_init == TRUE. + old = __kmp_signal( sig, SIG_DFL ); + __kmp_sighldrs[ sig ] = old; + __kmp_signal( sig, old ); + }; // if + KMP_MB(); /* Flush all pending memory write invalidates. */ +} // __kmp_install_one_handler + +static void +__kmp_remove_one_handler( int sig ) { + if ( __kmp_siginstalled[ sig ] ) { + sig_func_t old; + KMP_MB(); // Flush all pending memory write invalidates. + KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) ); + old = __kmp_signal( sig, __kmp_sighldrs[ sig ] ); + if ( old != __kmp_team_handler ) { + KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) ); + old = __kmp_signal( sig, old ); + }; // if + __kmp_sighldrs[ sig ] = NULL; + __kmp_siginstalled[ sig ] = 0; + KMP_MB(); // Flush all pending memory write invalidates. + }; // if +} // __kmp_remove_one_handler + + +void +__kmp_install_signals( int parallel_init ) +{ + KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) ); + if ( ! __kmp_handle_signals ) { + KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) ); + return; + }; // if + __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init ); + __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init ); + __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init ); + __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init ); + __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init ); + __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init ); +} // __kmp_install_signals + + +void +__kmp_remove_signals( void ) +{ + int sig; + KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) ); + for ( sig = 1; sig < NSIG; ++ sig ) { + __kmp_remove_one_handler( sig ); + }; // for sig +} // __kmp_remove_signals + + +#endif // KMP_HANDLE_SIGNALS + +/* Put the thread to sleep for a time period */ +void +__kmp_thread_sleep( int millis ) +{ + DWORD status; + + status = SleepEx( (DWORD) millis, FALSE ); + if ( status ) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( FunctionError, "SleepEx()" ), + KMP_ERR( error ), + __kmp_msg_null + ); + } +} + +/* Determine whether the given address is mapped into the current address space. */ +int +__kmp_is_address_mapped( void * addr ) +{ + DWORD status; + MEMORY_BASIC_INFORMATION lpBuffer; + SIZE_T dwLength; + + dwLength = sizeof(MEMORY_BASIC_INFORMATION); + + status = VirtualQuery( addr, &lpBuffer, dwLength ); + + return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) || + (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE ))); +} + +kmp_uint64 +__kmp_hardware_timestamp(void) +{ + kmp_uint64 r = 0; + + QueryPerformanceCounter((LARGE_INTEGER*) &r); + return r; +} + +/* Free handle and check the error code */ +void +__kmp_free_handle( kmp_thread_t tHandle ) +{ +/* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */ + BOOL rc; + rc = CloseHandle( tHandle ); + if ( !rc ) { + DWORD error = GetLastError(); + __kmp_msg( + kmp_ms_fatal, + KMP_MSG( CantCloseHandle ), + KMP_ERR( error ), + __kmp_msg_null + ); + } +} + +int +__kmp_get_load_balance( int max ) { + + static ULONG glb_buff_size = 100 * 1024; + + static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */ + static double glb_call_time = 0; /* Thread balance algorithm call time */ + + int running_threads = 0; // Number of running threads in the system. + NTSTATUS status = 0; + ULONG buff_size = 0; + ULONG info_size = 0; + void * buffer = NULL; + PSYSTEM_PROCESS_INFORMATION spi = NULL; + int first_time = 1; + + double call_time = 0.0; //start, finish; + + __kmp_elapsed( & call_time ); + + if ( glb_call_time && + ( call_time - glb_call_time < __kmp_load_balance_interval ) ) { + running_threads = glb_running_threads; + goto finish; + } + glb_call_time = call_time; + + // Do not spend time on running algorithm if we have a permanent error. + if ( NtQuerySystemInformation == NULL ) { + running_threads = -1; + goto finish; + }; // if + + if ( max <= 0 ) { + max = INT_MAX; + }; // if + + do { + + if ( first_time ) { + buff_size = glb_buff_size; + } else { + buff_size = 2 * buff_size; + } + + buffer = KMP_INTERNAL_REALLOC( buffer, buff_size ); + if ( buffer == NULL ) { + running_threads = -1; + goto finish; + }; // if + status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size ); + first_time = 0; + + } while ( status == STATUS_INFO_LENGTH_MISMATCH ); + glb_buff_size = buff_size; + + #define CHECK( cond ) \ + { \ + KMP_DEBUG_ASSERT( cond ); \ + if ( ! ( cond ) ) { \ + running_threads = -1; \ + goto finish; \ + } \ + } + + CHECK( buff_size >= info_size ); + spi = PSYSTEM_PROCESS_INFORMATION( buffer ); + for ( ; ; ) { + ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer ); + CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size ); + HANDLE pid = spi->ProcessId; + ULONG num = spi->NumberOfThreads; + CHECK( num >= 1 ); + size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 ); + CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer. + if ( spi->NextEntryOffset != 0 ) { + CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record. + }; // if + // pid == 0 corresponds to the System Idle Process. It always has running threads + // on all cores. So, we don't consider the running threads of this process. + if ( pid != 0 ) { + for ( int i = 0; i < num; ++ i ) { + THREAD_STATE state = spi->Threads[ i ].State; + // Count threads that have Ready or Running state. + // !!! TODO: Why comment does not match the code??? + if ( state == StateRunning ) { + ++ running_threads; + // Stop counting running threads if the number is already greater than + // the number of available cores + if ( running_threads >= max ) { + goto finish; + } + } // if + }; // for i + } // if + if ( spi->NextEntryOffset == 0 ) { + break; + }; // if + spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset ); + }; // forever + + #undef CHECK + + finish: // Clean up and exit. + + if ( buffer != NULL ) { + KMP_INTERNAL_FREE( buffer ); + }; // if + + glb_running_threads = running_threads; + + return running_threads; + +} //__kmp_get_load_balance() + |
