Initial futex support

Change-Id: I51a4f1117085ce23c7993c1a38e4124596636726
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/141
Tested-by: Jenkins Server
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

25 files changed, 634 insertions, 232 deletions

diff --git a/src/include/errno.h b/src/include/errno.h
new file mode 100644
index 000000000..664c3d46c
--- /dev/null
+++ b/src/include/errno.h
@@ -0,0 +1,7 @@
+#ifndef _ERRNO_H
+#define _ERRNO_H
+
+#define EAGAIN          11      // Try again
+#define	EWOULDBLOCK     EAGAIN  // operation would block
+
+#endif
diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index 87e769743..0f36dd941 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -9,7 +9,7 @@
 #include <kernel/types.h>
 #include <arch/ppc.H>
 #include <builtins.h>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 
 // Thread ID support only, Power7 (4 threads).
 #define KERNEL_MAX_SUPPORTED_CPUS (4 * 8 * 4) // Sockets, cores, threads.
@@ -36,7 +36,7 @@ struct cpu_t
     task_t* idle_task;
 
         /** XSCOM mutex to serialize access per CPU */
-    mutex_t xscom_mutex;
+    mutex_t * xscom_mutex;
 };
 
 /** @fn getCpuId
diff --git a/src/include/kernel/futexmgr.H b/src/include/kernel/futexmgr.H
new file mode 100644
index 000000000..824175136
--- /dev/null
+++ b/src/include/kernel/futexmgr.H
@@ -0,0 +1,89 @@
+#ifndef FUTEXMGR
+#define FUTEXMGR
+
+/**
+ * @file futexmgr.H
+ * @brief Declaration for kernel side futex management
+ */
+
+#include <stdint.h>
+#include <util/locked/list.H>
+#include <kernel/spinlock.H>
+
+struct task_t;
+
+
+/**
+ * @class FutexManager
+ * Kernel internal management of fuxtexs
+ */
+class FutexManager
+{
+    public:
+
+        /**
+         * Put the current processes on a wait queue
+         * @param[in] i_task  pointer to the current task structure
+         * @param[in] i_addr  Futex address 
+         * @param[in] i_val   Value that *i_addr should contain
+         * @returns [0 | error code]  if *i_addr != i_val returns EWOULDBLOCK
+         */
+        static uint64_t wait(task_t * i_task, uint64_t * i_addr, uint64_t i_val);
+
+        /**
+         * Wakeup threads
+         * @param[in] i_addr pointer to a futex
+         * @param[in] i_count The max number of threads to wake
+         * @returns The number of threads awoken
+         */
+        static uint64_t wake(uint64_t * i_addr, uint64_t i_count);
+
+    protected:
+
+        /**
+         * Ctor
+         */
+        FutexManager() {};
+
+        /**
+         * Dtor
+         */
+        ~FutexManager() {};
+
+    private: // functions
+
+        /**
+         * Put the current processes on a wait queue
+         * @param[in] i_task  pointer to the current task structure
+         * @param[in] i_addr  Futex address 
+         * @param[in] i_val   Value that *i_addr should contain
+         * @returns [0 | error code]  if *i_addr != i_val returns EWOULDBLOCK
+         */
+        uint64_t _wait(task_t * i_task, uint64_t * i_addr, uint64_t i_val);
+
+        /**
+         * Wakeup threads
+         * @param[in] i_addr pointer to a futex
+         * @param[in] i_count The max number of threads to wake
+         * @returns The number of threads awoken
+         */
+        uint64_t _wake(uint64_t * i_addr, uint64_t i_count);
+
+    private: // data
+
+        struct _FutexWait_t
+        {
+            _FutexWait_t * next;    ///< next _FutexWait_t in list
+            _FutexWait_t * prev;    ///< prev _FutexWait_t in list
+            uint64_t * key;         ///< search key is futex address
+            task_t* task;           ///< task on wait list
+        };
+
+        typedef Util::Locked::List<_FutexWait_t, uint64_t *> FutexList_t;
+
+        Spinlock iv_lock;       ///< lock
+        FutexList_t iv_list;    ///< List of waiting tasks for all futexes
+};
+
+#endif
+
diff --git a/src/include/kernel/syscalls.H b/src/include/kernel/syscalls.H
index 1b87c5019..26e2bdf75 100644
--- a/src/include/kernel/syscalls.H
+++ b/src/include/kernel/syscalls.H
@@ -24,15 +24,6 @@ namespace Systemcalls
             /** task_end() */
 	TASK_END,
             
-            /** mutex_create() */
-	MUTEX_CREATE,
-            /** mutex_destroy() */
-	MUTEX_DESTROY,
-            /** mutex_lock() */
-	MUTEX_LOCK_CONTESTED,
-            /** mutex_unlock() */
-	MUTEX_UNLOCK_CONTESTED,
-            
             /** msgq_create() */
 	MSGQ_CREATE,
             /** msgq_destroy() */
@@ -59,6 +50,12 @@ namespace Systemcalls
             /** nanosleep() */
 	TIME_NANOSLEEP,
 
+            /** futex_wait() */
+        FUTEX_WAIT,
+
+            /** futex_wake() */
+        FUTEX_WAKE,
+
 	SYSCALL_MAX
     };
 
diff --git a/src/include/kernel/usermutex.H b/src/include/kernel/usermutex.H
deleted file mode 100644
index c9f631579..000000000
--- a/src/include/kernel/usermutex.H
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef __KERNEL_USERMUTEX_H
-#define __KERNEL_USERMUTEX_H
-
-#include <util/locked/queue.H>
-#include <kernel/spinlock.H>
-#include <kernel/task.H>
-
-struct UserMutex
-{
-    uint64_t value;
-    bool unlock_pend;
-    Spinlock lock;
-    Util::Locked::Queue<task_t> waiting;
-
-    UserMutex() : value(0), unlock_pend(false) {};
-};
-
-#endif
diff --git a/src/include/sys/mmio.h b/src/include/sys/mmio.h
index c36a13821..3536071c8 100644
--- a/src/include/sys/mmio.h
+++ b/src/include/sys/mmio.h
@@ -2,7 +2,7 @@
 #define __SYS_MMIO_H
 
 #include <stdint.h>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 
 #ifdef __cplusplus
 extern "C"
@@ -38,7 +38,7 @@ int mmio_hmer_write(uint64_t value);
  *        affinity pin.  If the pin is moved the mutex is no longer
  *        guarenteed for the CPU the task is executing on.
  */
-mutex_t mmio_xscom_mutex();
+mutex_t * mmio_xscom_mutex();
 
 #ifdef __cplusplus
 }
diff --git a/src/include/sys/mutex.h b/src/include/sys/mutex.h
deleted file mode 100644
index 3eca65f06..000000000
--- a/src/include/sys/mutex.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#ifndef __SYS_MUTEX_H
-#define __SYS_MUTEX_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-typedef void* mutex_t;
-
-mutex_t mutex_create();
-int mutex_destroy(mutex_t);
-
-int mutex_lock(mutex_t);
-int mutex_unlock(mutex_t);
-
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/src/include/sys/sync.h b/src/include/sys/sync.h
new file mode 100644
index 000000000..68b570aec
--- /dev/null
+++ b/src/include/sys/sync.h
@@ -0,0 +1,89 @@
+#ifndef SYNC
+#define SYNC
+
+#include <stdint.h>
+
+/**
+ * Mutex object type
+ */
+struct _futex_imp_t
+{
+    uint64_t iv_val;
+};
+
+typedef _futex_imp_t mutex_t; 
+
+/**
+ * Barrier object type
+ */
+struct _barrier_imp_t
+{
+    mutex_t iv_mutex;
+    uint64_t iv_event;
+    uint64_t iv_missing;
+    uint64_t iv_count;
+};
+
+typedef _barrier_imp_t barrier_t;
+
+#define MUTEX_INITIALIZER {0}
+
+/**
+ * Initialize a barrier object
+ * @param[out] o_barrier The barrier
+ * @param[in] i_count The number of threads to wait on
+ * @pre an unitiailized barrier object
+ * @post a valid barrier object
+ */
+void barrier_init (barrier_t * o_barrier, uint64_t i_count);
+
+/**
+ * Destroy a barrier
+ * @param[in] i_barrier  The barrier
+ */
+void barrier_destroy (barrier_t * i_barrier);
+
+/**
+ * Wait on a barrier
+ * @param[in] i_barrier The barrier
+ * @post this thread will be blocked until the barrier count is reached.
+ */
+void barrier_wait (barrier_t * i_barrier);
+
+/**
+ * Create a mutex and initialize a mutex
+ * @returns a pointer to the mutex
+ */
+mutex_t * mutex_create();
+
+/**
+ * Initialize a mutex object
+ * @param[out] o_mutex the mutex
+ * @pre an uninitialized mutex object
+ * @post a valid mutex object
+ */
+void mutex_init(mutex_t * o_mutex);
+
+/**
+ * Destroy a mutex
+ * @param[in/out] i_mutex The mutex
+ * @pre mutex must have been created with mutex_create()
+ */
+void mutex_destroy(mutex_t *& io_mutex);
+
+/**
+ * Obtain a lock on a mutex
+ * @param[in] i_mutex The mutex
+ * @post returns when this thread has the lock
+ */
+void mutex_lock(mutex_t * i_mutex);
+
+/**
+ * Release a lock on a mutex
+ * @param[in] i_mutex the mutex
+ * @returns non zero on error
+ * @post mutex lock release
+ */
+void mutex_unlock(mutex_t * i_mutex);
+
+#endif
diff --git a/src/include/usr/trace/trace.H b/src/include/usr/trace/trace.H
index 722d31929..e4ea8922c 100644
--- a/src/include/usr/trace/trace.H
+++ b/src/include/usr/trace/trace.H
@@ -15,7 +15,7 @@
 #include <stdint.h>
 #include <trace/interface.H>
 #include <util/singleton.H>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 
 /******************************************************************************/
 // Globals/Constants
diff --git a/src/kernel/futexmgr.C b/src/kernel/futexmgr.C
new file mode 100644
index 000000000..aaae95b53
--- /dev/null
+++ b/src/kernel/futexmgr.C
@@ -0,0 +1,88 @@
+/**
+ * @file futexmgr.C
+ * @brief Definition for kernel side futex management
+ */
+#include <assert.h>
+#include <errno.h>
+#include <kernel/task.H>
+#include <kernel/cpumgr.H>
+#include <kernel/scheduler.H>
+#include <kernel/futexmgr.H>
+#include <util/singleton.H>
+
+
+//-----------------------------------------------------------------------------
+
+uint64_t FutexManager::wait(task_t* i_task, uint64_t * i_addr, uint64_t i_val)
+{
+    return Singleton<FutexManager>::instance()._wait(i_task, i_addr, i_val);
+}
+
+//-----------------------------------------------------------------------------
+
+uint64_t FutexManager::wake(uint64_t * i_addr, uint64_t i_count)
+{
+    return Singleton<FutexManager>::instance()._wake(i_addr, i_count);
+}
+
+//-----------------------------------------------------------------------------
+
+uint64_t FutexManager::_wait(task_t* i_task, uint64_t * i_addr, uint64_t i_val)
+{
+    uint64_t rc = 0;
+
+    iv_lock.lock();
+
+    if(*i_addr != i_val)
+    {
+        // some other thread has modified the futex
+        // bail-out retry required.
+        iv_lock.unlock();
+        rc = EWOULDBLOCK;
+    }
+    else
+    {
+        _FutexWait_t * waiter = new _FutexWait_t();
+        waiter->key = i_addr;
+        waiter->task = i_task;
+
+        // Now add the futex/task it to the wait queue
+        iv_list.insert(waiter);
+        iv_lock.unlock();
+        CpuManager::getCurrentCPU()->scheduler->setNextRunnable();
+    }
+
+    return rc;
+}
+
+//-----------------------------------------------------------------------------
+
+uint64_t FutexManager::_wake(uint64_t * i_addr, uint64_t i_count)
+{
+    uint64_t started = 0;
+
+    // Remove task(s) from futex queue
+    // Put it/them on the run queue
+    iv_lock.lock();
+    while(started < i_count)
+    {
+        _FutexWait_t * waiter = iv_list.find(i_addr);
+        if(waiter == NULL)
+        {
+            break;
+        }
+
+        task_t * wait_task = waiter->task;
+        iv_list.erase(waiter);
+
+        // This means we had a waiter in the queue, but that waiter had
+        // a Null task assigned to it.  This should NEVER happen
+        assert(wait_task != NULL);
+
+        wait_task->cpu->scheduler->addTask(wait_task);
+        ++started;
+    }
+    iv_lock.unlock();
+
+    return started;
+}
diff --git a/src/kernel/makefile b/src/kernel/makefile
index 057552b9d..4f100c660 100644
--- a/src/kernel/makefile
+++ b/src/kernel/makefile
@@ -1,7 +1,7 @@
 ROOTPATH = ../..
 
 OBJS = start.o kernel.o console.o pagemgr.o heapmgr.o taskmgr.o cpumgr.o
-OBJS += syscall.o scheduler.o spinlock.o exception.o vmmmgr.o timemgr.o
+OBJS += syscall.o scheduler.o spinlock.o exception.o vmmmgr.o timemgr.o futexmgr.o
 
 include ${ROOTPATH}/config.mk
 
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index 837ed3d21..85252a408 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -6,9 +6,9 @@
 #include <kernel/syscalls.H>
 #include <kernel/console.H>
 #include <kernel/pagemgr.H>
-#include <kernel/usermutex.H>
 #include <kernel/msg.H>
 #include <kernel/timemgr.H>
+#include <kernel/futexmgr.H>
 
 extern "C"
 void kernel_execute_decrementer()
@@ -26,10 +26,6 @@ namespace Systemcalls
     void TaskStart(task_t*);
     void TaskEnd(task_t*);
     void TaskGettid(task_t*);
-    void MutexCreate(task_t*);
-    void MutexDestroy(task_t*);
-    void MutexLockCont(task_t*);
-    void MutexUnlockCont(task_t*);
     void MsgQCreate(task_t*);
     void MsgQDestroy(task_t*);
     void MsgQRegisterRoot(task_t*);
@@ -41,6 +37,8 @@ namespace Systemcalls
     void MmioMap(task_t*);
     void MmioUnmap(task_t*);
     void TimeNanosleep(task_t*);
+    void FutexWait(task_t *t);
+    void FutexWake(task_t *t);
 
     syscall syscalls[] =
 	{
@@ -48,11 +46,6 @@ namespace Systemcalls
 	    &TaskStart,
 	    &TaskEnd,
 
-	    &MutexCreate,
-	    &MutexDestroy,
-	    &MutexLockCont,
-	    &MutexUnlockCont,
-
 	    &MsgQCreate,
 	    &MsgQDestroy,
 	    &MsgQRegisterRoot,
@@ -67,6 +60,9 @@ namespace Systemcalls
 	    &MmioUnmap,
 
 	    &TimeNanosleep,
+
+            &FutexWait,
+            &FutexWake,
 	};
 };
 
@@ -133,62 +129,6 @@ namespace Systemcalls
 	delete t;
     }
 
-    void MutexCreate(task_t* t)
-    {
-	UserMutex * m = new UserMutex();
-	TASK_SETRTN(t, (uint64_t)m);
-    }
-
-    void MutexDestroy(task_t* t)
-    {
-	// TODO: Extra verification of parameter and mutex state.
-
-	delete (UserMutex*) TASK_GETARG0(t);
-	TASK_SETRTN(t, 0);
-    }
-
-    void MutexLockCont(task_t* t)
-    {
-	// TODO: Extra verification of parameter and mutex state.
-	
-	UserMutex* m = (UserMutex*) TASK_GETARG0(t);
-	m->lock.lock();
-	if (m->unlock_pend) 
-	{   
-	    // We missed the unlock syscall, take lock and return to userspace.
-	    m->unlock_pend = false;
-	    m->lock.unlock();
-
-	}
-	else
-	{
-	    // Queue ourself to wait for unlock.
-	    m->waiting.insert(TaskManager::getCurrentTask());
-	    m->lock.unlock();
-	    CpuManager::getCurrentCPU()->scheduler->setNextRunnable();
-	}
-	TASK_SETRTN(t, 0);	
-    }
-
-    void MutexUnlockCont(task_t* t)
-    {
-	// TODO: Extra verification of parameter and mutex state.
-
-	UserMutex* m = (UserMutex*) TASK_GETARG0(t);
-	m->lock.lock();
-	task_t* wait_task = m->waiting.remove();
-	if (NULL == wait_task)
-	{
-	    m->unlock_pend = true;
-	}
-	else
-	{
-	    wait_task->cpu->scheduler->addTask(wait_task);
-	}
-	m->lock.unlock();
-	TASK_SETRTN(t, 0);
-    }
-
     void MsgQCreate(task_t* t)
     {
 	TASK_SETRTN(t, (uint64_t) new MessageQueue());
@@ -349,5 +289,37 @@ namespace Systemcalls
 	s->setNextRunnable();
     }
 
+    /**
+     * Put task on wait queue based on futex
+     * @param[in] t:  The task to block
+     */
+    void FutexWait(task_t * t)
+    {
+        uint64_t * uaddr = (uint64_t *) TASK_GETARG0(t);
+        uint64_t val   = (uint64_t) TASK_GETARG1(t);
+
+        //TODO translate uaddr from user space to kernel space
+        //     Right now they are the same.
+
+        uint64_t rc = FutexManager::wait(t,uaddr,val);
+        TASK_SETRTN(t,rc);
+    }
+
+    /**
+     * Wake tasks on futex wait queue
+     * @param[in] t:  The current task
+     */
+    void FutexWake(task_t * t)
+    {
+        uint64_t * uaddr = (uint64_t *) TASK_GETARG0(t);
+        uint64_t count = (uint64_t) TASK_GETARG1(t);
+
+        // TODO translate uaddr from user space to kernel space
+        //      Right now they are the same
+
+        uint64_t started = FutexManager::wake(uaddr,count);
+
+        TASK_SETRTN(t,started);
+    }
 
 };
diff --git a/src/lib/makefile b/src/lib/makefile
index d15ca3bef..3fdb3a8b1 100644
--- a/src/lib/makefile
+++ b/src/lib/makefile
@@ -1,7 +1,9 @@
 ROOTPATH = ../..
 
 OBJS = string.o stdlib.o assert.o stdio.o
-OBJS += syscall_stub.o syscall_task.o syscall_mutex.o syscall_msg.o
-OBJS += syscall_mmio.o syscall_time.o
+OBJS += syscall_stub.o syscall_task.o syscall_msg.o
+OBJS += syscall_mmio.o syscall_time.o sync.o
+
+SUBDIRS = test.d
 
 include ${ROOTPATH}/config.mk
diff --git a/src/lib/sync.C b/src/lib/sync.C
new file mode 100644
index 000000000..a70761eef
--- /dev/null
+++ b/src/lib/sync.C
@@ -0,0 +1,160 @@
+#include <sys/sync.h>
+#include <sys/syscall.h>
+#include <assert.h>
+
+using namespace Systemcalls;
+
+//-----------------------------------------------------------------------------
+
+uint64_t futex_wait(uint64_t * i_addr, uint64_t i_val)
+{
+    return (uint64_t) _syscall2(FUTEX_WAIT,i_addr, (void *)i_val);
+}
+
+//-----------------------------------------------------------------------------
+
+uint64_t futex_wake(uint64_t * i_addr, uint64_t i_count)
+{
+    return (uint64_t) _syscall2(FUTEX_WAKE, i_addr, (void *)i_count);
+}
+
+//-----------------------------------------------------------------------------
+
+void barrier_init (barrier_t * o_barrier, uint64_t i_count)
+{
+    mutex_init(&(o_barrier->iv_mutex));
+    o_barrier->iv_event = 0;
+    o_barrier->iv_count = i_count;
+    o_barrier->iv_missing = i_count;
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+void barrier_destroy (barrier_t * i_barrier)
+{
+    assert(i_barrier->iv_missing == i_barrier->iv_count);
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+void barrier_wait (barrier_t * i_barrier)
+{
+    mutex_lock(&(i_barrier->iv_mutex));
+    --(i_barrier->iv_missing);
+    if(i_barrier->iv_missing > 0)
+    {
+        uint64_t l_event = i_barrier->iv_event;
+        mutex_unlock(&(i_barrier->iv_mutex));
+        do
+        {
+            futex_wait(&(i_barrier->iv_event), l_event);
+        } while (i_barrier->iv_event == l_event);
+    }
+    else
+    {
+        ++(i_barrier->iv_event);
+        i_barrier->iv_missing = i_barrier->iv_count;
+
+        // Wake em all up
+        futex_wake(&(i_barrier->iv_event), UINT64_MAX);
+        mutex_unlock(&(i_barrier->iv_mutex));
+    }
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+mutex_t * mutex_create()
+{
+    mutex_t * m = new mutex_t;
+    mutex_init(m);
+    return m;
+}
+
+//-----------------------------------------------------------------------------
+
+void mutex_init(mutex_t * o_mutex)
+{
+    o_mutex->iv_val = 0;
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+void mutex_destroy(mutex_t *& i_mutex)
+{
+    delete i_mutex;
+    i_mutex = NULL;
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+void mutex_lock(mutex_t * i_mutex)
+{
+#ifdef __IDEA_1
+    uint64_t l_count = __sync_val_compare_and_swap(&(i_mutex->iv_val),0,1);
+
+    if(l_count != 0)
+    {
+        if (l_count != 2)
+            l_count = __sync_lock_test_and_set(&(i_mutex->iv_val), 2);
+
+        while( l_count != 0 )
+        {
+            futex_wait( &(i_mutex->iv_val), 2);
+            l_count = __sync_lock_test_and_set(&(i_mutex->iv_val),2);
+            // if more than one thread gets out - one continues while
+            // the rest get blocked again.
+        }
+    }
+#endif
+
+    // Idea # 2
+    while(1)
+    {
+        uint64_t l_count = __sync_val_compare_and_swap(&(i_mutex->iv_val),0,1);
+        if( 0 == l_count ) // uncontended lock
+        {
+            break;
+        }
+        else if ( 2 == l_count ) // already contended
+        {
+            futex_wait( &(i_mutex->iv_val), l_count);
+        }
+        else if ( 1 == l_count ) // lock now contended
+        {
+            if (__sync_bool_compare_and_swap(&(i_mutex->iv_val),1,2))
+            {
+                futex_wait( &(i_mutex->iv_val), 2);
+            }
+        }
+    }
+
+    return;
+}
+
+//-----------------------------------------------------------------------------
+
+void mutex_unlock(mutex_t * i_mutex)
+{
+#ifdef __IDEA_1
+    if (__sync_fetch_and_sub(&(i_mutex->iv_val), 1) != 1)
+    {
+        i_mutex->iv_val = 0; // the thread that is started will set this back to 2
+        futex_wake(&(i_mutex->iv_val), 1); // wake one thread
+    }
+#endif
+    // #idea 2
+    uint64_t l_count = __sync_lock_test_and_set(&(i_mutex->iv_val),0);
+    if(2 == l_count)
+    {
+        futex_wake(&(i_mutex->iv_val), 1); // wake one thread
+    }
+ 
+    return;
+}
+
+
diff --git a/src/lib/syscall_mmio.C b/src/lib/syscall_mmio.C
index 83498798b..91e4e113b 100644
--- a/src/lib/syscall_mmio.C
+++ b/src/lib/syscall_mmio.C
@@ -26,7 +26,7 @@ int mmio_hmer_write(uint64_t value)
     return (int)(int64_t)_syscall1(MMIO_HMER_WRITE, (void*)value);
 }
 
-mutex_t mmio_xscom_mutex()
+mutex_t * mmio_xscom_mutex()
 {
     // Get task structure.
     register task_t* task;
@@ -36,7 +36,7 @@ mutex_t mmio_xscom_mutex()
     assert(task->affinity_pinned);
 
     // Get mutex from cpu structure.
-    mutex_t mutex = task->cpu->xscom_mutex;
+    mutex_t * mutex = task->cpu->xscom_mutex;
 
     // Create mutex if not created.
     if (NULL == mutex)
diff --git a/src/lib/syscall_mutex.C b/src/lib/syscall_mutex.C
deleted file mode 100644
index 8ac392792..000000000
--- a/src/lib/syscall_mutex.C
+++ /dev/null
@@ -1,33 +0,0 @@
-#include <sys/mutex.h>
-#include <sys/syscall.h>
-#include <kernel/usermutex.H>
-
-using namespace Systemcalls;
-
-mutex_t mutex_create()
-{
-    return (mutex_t) _syscall0(MUTEX_CREATE);
-}
-
-int mutex_destroy(mutex_t m)
-{
-    return (int64_t)_syscall1(MUTEX_DESTROY, m);
-}
-
-int mutex_lock(mutex_t m)
-{
-    uint64_t oldvalue = __sync_fetch_and_add(&((UserMutex*)m)->value, 1);
-    if (0 == oldvalue)
-	return 0;
-    else 
-	return (int64_t)_syscall1(MUTEX_LOCK_CONTESTED, m);
-}
-
-int mutex_unlock(mutex_t m)
-{
-    uint64_t oldvalue = __sync_fetch_and_sub(&((UserMutex*)m)->value, 1);
-    if (1 == oldvalue)
-	return 0;
-    else
-	return (int64_t)_syscall1(MUTEX_UNLOCK_CONTESTED, m);
-}
diff --git a/src/lib/test/makefile b/src/lib/test/makefile
new file mode 100644
index 000000000..723f4b36b
--- /dev/null
+++ b/src/lib/test/makefile
@@ -0,0 +1,6 @@
+ROOTPATH = ../../..
+
+MODULE = testsyslib
+TESTS = *.H
+
+include ${ROOTPATH}/config.mk
diff --git a/src/lib/test/synctest.H b/src/lib/test/synctest.H
new file mode 100644
index 000000000..5a53d2e0a
--- /dev/null
+++ b/src/lib/test/synctest.H
@@ -0,0 +1,93 @@
+#ifndef __SYNCTEST_H
+#define __SYNCTEST_H
+/**
+ *  @file synctest.H
+ *
+ *  @brief Test cases for the sycronization
+*/
+
+#include <cxxtest/TestSuite.H>
+#include <sys/sync.h>
+#include <sys/task.h>
+
+class SyncTest: public CxxTest::TestSuite
+{
+    public:
+
+
+        void testMutex()
+        {
+            mutex_init(&mutex);
+            barrier_init(&barrier, 7);
+
+            task_create(func1, this);
+            task_create(func2, this);
+            task_create(func2, this);
+            task_create(func2, this);
+            task_create(func2, this);
+            task_create(func2, this);
+            barrier_wait(&barrier);
+            TS_TRACE("ALL THREADS ENDED");
+        }
+
+        void testBarrier()
+        {
+            barrier_t barrier;
+            barrier_init(&barrier,3);
+            task_create(func3,&barrier);
+            task_create(func4,&barrier);
+            barrier_wait(&barrier);
+            TS_TRACE("B0");
+            barrier_destroy(&barrier);
+        }
+
+    private:
+
+        mutex_t mutex;
+        barrier_t barrier;
+
+        static void func1(void * i_p)
+        {
+            SyncTest * my = (SyncTest *) i_p;
+            mutex_t * mutex = &(my->mutex);
+            barrier_t * barrier = &(my->barrier);
+
+            mutex_lock(mutex);
+            for(int i = 0; i < 100000; ++i);
+            TS_TRACE("ME FIRST");
+            mutex_unlock(mutex);
+            barrier_wait(barrier);
+            task_end();
+        }
+
+        static void func2(void * i_p)
+        {
+            SyncTest * my = (SyncTest *) i_p;
+            mutex_t * mutex = &(my->mutex);
+            barrier_t * barrier = &(my->barrier);
+
+            mutex_lock(mutex);
+            TS_TRACE("ME NEXT");
+            mutex_unlock(mutex);
+            barrier_wait(barrier);
+            task_end();
+        }
+
+        static void func3(void * i_p)
+        {
+            barrier_t * barrier = (barrier_t *) i_p;
+            barrier_wait(barrier);
+            TS_TRACE("B1");
+            task_end();
+        }
+        static void func4(void * i_p)
+        {
+            barrier_t * barrier = (barrier_t *) i_p;
+            barrier_wait(barrier);
+            TS_TRACE("B2");
+            task_end();
+        }
+
+};
+#endif
+
diff --git a/src/makefile b/src/makefile
index 4a1987678..f3bb5dfe8 100644
--- a/src/makefile
+++ b/src/makefile
@@ -9,16 +9,16 @@ BASE_OBJECTS = console.o spinlock.o string.o stdlib.o assert.o stdio.o \
 
 DIRECT_BOOT_OBJECTS = start.o kernel.o taskmgr.o cpumgr.o syscall.o \
 		      scheduler.o exception.o vmmmgr.o timemgr.o \
-		      syscall_stub.o syscall_task.o syscall_mutex.o \
+		      syscall_stub.o syscall_task.o \
 		      syscall_msg.o syscall_mmio.o syscall_time.o \
-		      init_main.o vfs_main.o
+		      init_main.o vfs_main.o sync.o futexmgr.o
 
 RUNTIME_OBJECTS =
 
 BASE_MODULES = trace errl devicefw
 DIRECT_BOOT_MODULES = example
 RUNTIME_MODULES =
-TESTCASE_MODULES = cxxtest testerrl testdevicefw
+TESTCASE_MODULES = cxxtest testerrl testdevicefw testsyslib
 
 RELOCATABLE_IMAGE_LDFLAGS = -pie --export-dynamic
 
diff --git a/src/sys/init/init_main.C b/src/sys/init/init_main.C
index aae7f729e..dd83ea131 100644
--- a/src/sys/init/init_main.C
+++ b/src/sys/init/init_main.C
@@ -4,18 +4,18 @@
 #include <sys/syscall.h>
 
 #include <sys/task.h>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 #include <sys/msg.h>
 #include <sys/mmio.h>
 
-mutex_t global_mutex;
+mutex_t global_mutex = MUTEX_INITIALIZER;
 
 /*
 void init_child(void* unused)
 {
-    mutex_lock(global_mutex);
+    mutex_lock(&global_mutex);
     printk("Crun: %d on %d\n", task_gettid(), task_getcpuid());
-    mutex_unlock(global_mutex);
+    mutex_unlock(&global_mutex);
     for (volatile int i = 0 ; i < 100000; i++);
     task_end();
 }
@@ -39,8 +39,6 @@ void init_main(void* unused)
     uint64_t* mmio_addr = (uint64_t*) mmio_map((void*)0x800000000, 1);
     printk("MMIO Access %lx\n", *mmio_addr);
 
-    global_mutex = mutex_create();
-
     msg_q_t msgq = msg_q_create();
     msg_q_register(msgq, "/msg/init");
 
@@ -53,11 +51,11 @@ void init_main(void* unused)
 
     while(1)
     {
-	mutex_lock(global_mutex);
+	mutex_lock(&global_mutex);
 	int t = task_create(&init_child, NULL);
 	printk("Create child %d\n", t);
 	for (volatile int i = 0 ; i < 1000000; i++);
-	mutex_unlock(global_mutex);
+	mutex_unlock(&global_mutex);
     }
 */
 
diff --git a/src/usr/cxxtest/cxxtestgen.pl b/src/usr/cxxtest/cxxtestgen.pl
index 8fa5ab5d7..24b722163 100755
--- a/src/usr/cxxtest/cxxtestgen.pl
+++ b/src/usr/cxxtest/cxxtestgen.pl
@@ -613,7 +613,7 @@ sub writeHostBootPreamble() {
   ##    print the standard includes for Hostboot
   print "\n";
   print "#include <kernel/console.H>\n";
-  print "#include <sys/mutex.h>\n";
+  print "#include <sys/sync.h>\n";
   print "#include <sys/vfs.h>\n";
   print "#include <sys/task.h>\n";
   print "#include <trace/interface.H>\n";
diff --git a/src/usr/devicefw/associator.C b/src/usr/devicefw/associator.C
index d076c311e..e09852bf9 100644
--- a/src/usr/devicefw/associator.C
+++ b/src/usr/devicefw/associator.C
@@ -13,17 +13,16 @@ namespace DeviceFW
     trace_desc_t* g_traceBuffer = NULL;
     TRAC_INIT(&g_traceBuffer, "DevFW", 4096);
 
-    Associator::Associator() : iv_mutex(mutex_create())
+    Associator::Associator() : iv_mutex()
     {
         TRACFCOMP(g_traceBuffer, ENTER_MRK "Associator::Associator");
-        
+        mutex_init(&iv_mutex);
         // Allocate first level of map (access types).
         iv_routeMap = iv_associations.allocate(LAST_DRIVER_ACCESS_TYPE);
     }
 
     Associator::~Associator()
     {
-        mutex_destroy(iv_mutex);
 
         TRACFCOMP(g_traceBuffer, EXIT_MRK "Associator::~Associator");        
     }
@@ -40,7 +39,7 @@ namespace DeviceFW
         // compiler due to the template specializations in driverif.H.  
         // No assert-checks will be done here.
 
-        mutex_lock(iv_mutex);
+        mutex_lock(&iv_mutex);
         
         size_t ops = 0;
         AssociationData targets = AssociationData();
@@ -118,14 +117,13 @@ namespace DeviceFW
         // Set function offset into map.  True flag indicates valid.
         (*iv_associations[targets.offset]) = AssociationData(true, opLoc);
 
-        mutex_unlock(iv_mutex);
+        mutex_unlock(&iv_mutex);
     }
 
     errlHndl_t Associator::performOp(OperationType i_opType,
                                      TargetHandle_t i_target,
                                      void* io_buffer, size_t& io_buflen,
                                      int64_t i_accessType, va_list i_addr) 
-        const
     {
         TRACDCOMP(g_traceBuffer, "Device op requested for (%d, %d, %d)",
                   i_opType, i_accessType, /*TODO: i_target->type*/PROCESSOR);
@@ -136,7 +134,7 @@ namespace DeviceFW
         
         errlHndl_t l_errl = NULL;
         
-        mutex_lock(iv_mutex);
+        mutex_lock(&iv_mutex);
 
             // Function pointer found for this route request.
         deviceOp_t l_devRoute = NULL;
@@ -203,7 +201,7 @@ namespace DeviceFW
             }
         } while(0);
 
-        mutex_unlock(iv_mutex);        
+        mutex_unlock(&iv_mutex);        
         
         // Call function if one was found, create error otherwise.
         if (NULL == l_devRoute)
diff --git a/src/usr/devicefw/associator.H b/src/usr/devicefw/associator.H
index 2052781b7..aba457338 100644
--- a/src/usr/devicefw/associator.H
+++ b/src/usr/devicefw/associator.H
@@ -5,7 +5,7 @@
 #define __DEVICEFW_ASSOCIATOR_H
 
 #include <devicefw/driverif.H>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 #include <vector>
 
 #include "assoccontain.H"
@@ -53,7 +53,7 @@ namespace DeviceFW
             errlHndl_t performOp(OperationType i_opType,
                                  TargetHandle_t i_target,
                                  void* io_buffer, size_t& io_buflen,
-                                 int64_t i_accessType, va_list i_addr) const;
+                                 int64_t i_accessType, va_list i_addr);
         private:
             typedef std::vector<deviceOp_t> opVector_t;
             
diff --git a/src/usr/example/example.C b/src/usr/example/example.C
index 536100368..632eb6882 100644
--- a/src/usr/example/example.C
+++ b/src/usr/example/example.C
@@ -1,5 +1,5 @@
 #include <kernel/console.H>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 #include <sys/vfs.h>
 #include <sys/task.h>
 #include <trace/interface.H>
@@ -7,7 +7,7 @@
 #include <example/examplerc.H>
 #include <errl/errlentry.H>
 
-//static mutex_t value = mutex_create();
+//static mutex_t value = MUTEX_INITIALIZER;
 trace_desc_t *g_trac_test = NULL;
 TRAC_INIT(&g_trac_test, "EXAMPLE", 4096);
 
diff --git a/src/usr/trace/trace.C b/src/usr/trace/trace.C
index e7687ad7e..569af1e08 100644
--- a/src/usr/trace/trace.C
+++ b/src/usr/trace/trace.C
@@ -18,7 +18,7 @@
 #include <limits.h>
 #include <stdlib.h>
 #include <sys/task.h>
-#include <sys/mutex.h>
+#include <sys/sync.h>
 #include <string.h>
 
 #include <trace/trace.H>
@@ -57,7 +57,7 @@ Trace& Trace::getTheInstance()
 /******************************************************************************/
 Trace::Trace()
 {
-    iv_trac_mutex = mutex_create();
+    mutex_init(&iv_trac_mutex);
 
     g_trac_global = static_cast<trace_desc_t *>(malloc(TRAC_DEFAULT_BUFFER_SIZE));
 
@@ -137,7 +137,6 @@ void Trace::trace_adal_write_all(trace_desc_t *io_td,
     /*------------------------------------------------------------------------*/
     /*  Local Variables                                                       */
     /*------------------------------------------------------------------------*/
-    int64_t                 l_rc = 0;
     uint32_t                l_entry_size = 0;
     trace_entire_entry_t    l_entry;
 
@@ -206,27 +205,15 @@ void Trace::trace_adal_write_all(trace_desc_t *io_td,
         // buffer for this
 
         // CRITICAL REGION START
-        l_rc = mutex_lock(iv_trac_mutex);
-        if(l_rc != 0)
-        {
-            // This will assert so we'll never hit this code.
-        }
-        else
-        {
-            // Update the entry count
-            io_td->te_count++;
+        mutex_lock(&iv_trac_mutex);
+        // Update the entry count
+        io_td->te_count++;
 
-            writeData(io_td,
-                      static_cast<void *>(&l_entry),
-                      l_entry_size);
+        writeData(io_td,
+                  static_cast<void *>(&l_entry),
+                  l_entry_size);
 
-            l_rc = mutex_unlock(iv_trac_mutex);
-            if(l_rc != 0)
-            {
-                // Badness
-                // This will assert so we'll never hit this code.
-            }
-        }
+        mutex_unlock(&iv_trac_mutex);
         // CRITICAL REGION END
     }
 
@@ -245,7 +232,6 @@ void Trace::trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash
     /*------------------------------------------------------------------------*/
     /*  Local Variables                                                       */
     /*------------------------------------------------------------------------*/
-    int64_t                 l_rc = 0;
     uint32_t                l_entry_size = 0;
     trace_bin_entry_t       l_entry;
 
@@ -291,40 +277,28 @@ void Trace::trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash
         convertTime(&l_entry.stamp);
 
         // CRITICAL REGION START
-        l_rc = mutex_lock(iv_trac_mutex);
-        if(l_rc != 0)
-        {
-            // This will assert so we'll never hit this code.
-        }
-        else
-        {
+        mutex_lock(&iv_trac_mutex);
 
-            // Increment trace counter
-            io_td->te_count++;
+        // Increment trace counter
+        io_td->te_count++;
 
-            // First write the header
-            writeData(io_td,
-                      static_cast<void *>(&l_entry),
-                      sizeof(l_entry));
+        // First write the header
+        writeData(io_td,
+                  static_cast<void *>(&l_entry),
+                  sizeof(l_entry));
 
-            // Now write the actual binary data
-            writeData(io_td,
-                      i_ptr,
-                      i_size);
+        // Now write the actual binary data
+        writeData(io_td,
+                  i_ptr,
+                  i_size);
 
-            // Now write the size at the end
-            writeData(io_td,
-                      static_cast<void *>(&l_entry_size),
-                      sizeof(l_entry_size));
+        // Now write the size at the end
+        writeData(io_td,
+                  static_cast<void *>(&l_entry_size),
+                  sizeof(l_entry_size));
 
-            // CRITICAL REGION END
-            l_rc = mutex_unlock(iv_trac_mutex);
-            if(l_rc != 0)
-            {
-                // Badness
-                // This will assert so we'll never hit this code.
-            }
-        }
+        // CRITICAL REGION END
+        mutex_unlock(&iv_trac_mutex);
 
     }while(0);