/* IBM_PROLOG_BEGIN_TAG */ /* This is an automatically generated prolog. */ /* */ /* $Source: src/lib/sync.C $ */ /* */ /* IBM CONFIDENTIAL */ /* */ /* COPYRIGHT International Business Machines Corp. 2011,2014 */ /* */ /* p1 */ /* */ /* Object Code Only (OCO) source materials */ /* Licensed Internal Code Source Materials */ /* IBM HostBoot Licensed Internal Code */ /* */ /* The source code for this program is not published or otherwise */ /* divested of its trade secrets, irrespective of what has been */ /* deposited with the U.S. Copyright Office. */ /* */ /* Origin: 30 */ /* */ /* IBM_PROLOG_END_TAG */ #include #include #include #include #include #include using namespace Systemcalls; //----------------------------------------------------------------------------- int futex_wait(uint64_t * i_addr, uint64_t i_val) { return (int64_t) _syscall5(SYS_FUTEX, (void *)FUTEX_WAIT, i_addr, (void *)i_val, NULL, NULL); } //----------------------------------------------------------------------------- int futex_wake(uint64_t * i_addr, uint64_t i_count) { return (int64_t) _syscall5(SYS_FUTEX, (void *)FUTEX_WAKE, i_addr, (void *)i_count, NULL, NULL); } //----------------------------------------------------------------------------- int64_t futex_requeue(uint64_t * i_addr, uint64_t i_count1, uint64_t i_count2, uint64_t * i_futex2) { return (int64_t) _syscall5(SYS_FUTEX, (void *)FUTEX_REQUEUE, i_addr, (void *)i_count1, (void *)i_count2, i_futex2); } //----------------------------------------------------------------------------- void barrier_init (barrier_t * o_barrier, uint64_t i_count) { mutex_init(&(o_barrier->iv_mutex)); o_barrier->iv_event = 0; o_barrier->iv_count = i_count; o_barrier->iv_missing = i_count; return; } //----------------------------------------------------------------------------- void barrier_destroy (barrier_t * i_barrier) { crit_assert(i_barrier->iv_missing == i_barrier->iv_count); return; } //----------------------------------------------------------------------------- void barrier_wait (barrier_t * i_barrier) { mutex_lock(&(i_barrier->iv_mutex)); --(i_barrier->iv_missing); if(i_barrier->iv_missing > 0) { uint64_t l_event = i_barrier->iv_event; mutex_unlock(&(i_barrier->iv_mutex)); do { futex_wait(&(i_barrier->iv_event), l_event); } while (i_barrier->iv_event == l_event); } else { ++(i_barrier->iv_event); i_barrier->iv_missing = i_barrier->iv_count; // Wake em all up futex_wake(&(i_barrier->iv_event), UINT64_MAX); mutex_unlock(&(i_barrier->iv_mutex)); } return; } //----------------------------------------------------------------------------- void mutex_init(mutex_t * o_mutex) { o_mutex->iv_val = 0; return; } //----------------------------------------------------------------------------- void mutex_destroy(mutex_t * i_mutex) { i_mutex->iv_val = ~0; return; } //----------------------------------------------------------------------------- void mutex_lock(mutex_t * i_mutex) { // Weak consistency notes: // Since this is a lock, we do not need to ensure that all writes // are globally visible prior to execution (lwsync) but we do need // to ensure that all instructions finish completion prior to // leaving (isync). Both __sync_val_compare_and_swap and // __sync_lock_test_and_set have an implied isync. uint64_t l_count = __sync_val_compare_and_swap(&(i_mutex->iv_val),0,1); if(unlikely(l_count != 0)) { if (likely(l_count != 2)) { l_count = __sync_lock_test_and_set(&(i_mutex->iv_val), 2); } while( l_count != 0 ) { futex_wait( &(i_mutex->iv_val), 2); l_count = __sync_lock_test_and_set(&(i_mutex->iv_val),2); // if more than one task gets out - one continues while // the rest get blocked again. } } return; } //----------------------------------------------------------------------------- void mutex_unlock(mutex_t * i_mutex) { // Weak consistency notes: // Since this is an unlock we need to ensure that all writes are // globally visible prior to execution (lwsync). The // __sync_fetch_and_sub has an implied lwsync. If we need to // release a task, due to a contended mutex, the write to iv_val // and futex_wake pair will appear globally ordered due to the // context synchronizing nature of the 'sc' instruction. uint64_t l_count = __sync_fetch_and_sub(&(i_mutex->iv_val),1); if(unlikely(2 <= l_count)) { i_mutex->iv_val = 0; futex_wake(&(i_mutex->iv_val), 1); // wake one task } return; } void sync_cond_init(sync_cond_t * i_cond) { i_cond->mutex = NULL; i_cond->sequence = 0; } void sync_cond_destroy(sync_cond_t * i_cond) { // don't need to do anything } int sync_cond_wait(sync_cond_t * i_cond, mutex_t * i_mutex) { uint64_t seq = i_cond->sequence; if(i_cond->mutex != i_mutex) { if(i_cond->mutex) return EINVAL; // Atomically set mutex __sync_bool_compare_and_swap(&i_cond->mutex, NULL, i_mutex); if(i_cond->mutex != i_mutex) return EINVAL; } mutex_unlock(i_mutex); futex_wait( &(i_cond->sequence), seq); // Can't continue until i_mutex lock is obtained. // Note: // mutex_lock(i_mutex); <--- This does not work // We have to mark the mutex as contended '2' because there is a race // condition between this thread (eventually) calling mutex_unlock and // the kernel moving tasks from the condition futex to the mutex futex // during a sync_condition_broadcast. By marking contended, we force the // subsequent mutex_unlock to call to the kernel which will obtain a // spinlock to ensure all tasks have been moved to the mutex_futex. while(0 != __sync_lock_test_and_set(&(i_mutex->iv_val), 2)) { futex_wait(&(i_mutex->iv_val),2); } return 0; } void sync_cond_signal(sync_cond_t * i_cond) { __sync_fetch_and_add(&(i_cond->sequence),1); // Wake up one futex_wake(&(i_cond->sequence), 1); } void sync_cond_broadcast(sync_cond_t * i_cond) { mutex_t * m = i_cond->mutex; // no mutex means no waiters if(!m) return; // wake up all __sync_fetch_and_add(&(i_cond->sequence),1); // need to wake up one on the sequence and // re-queue the rest onto the mutex m; futex_requeue(&(i_cond->sequence),1,UINT64_MAX,&(m->iv_val)); }