Create per-CPU mutex for XSCOM usage.

Change-Id: I76ad5c1d553d544b52910afd82e2716781ea13b9
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/140
Tested-by: Jenkins Server
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>

4 files changed, 55 insertions, 0 deletions

diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index 0fb794989..87e769743 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -9,6 +9,7 @@
 #include <kernel/types.h>
 #include <arch/ppc.H>
 #include <builtins.h>
+#include <sys/mutex.h>
 
 // Thread ID support only, Power7 (4 threads).
 #define KERNEL_MAX_SUPPORTED_CPUS (4 * 8 * 4) // Sockets, cores, threads.
@@ -33,6 +34,9 @@ struct cpu_t
     void* scheduler_extra;
         /** Pointer to the idle task for this CPU */
     task_t* idle_task;
+
+        /** XSCOM mutex to serialize access per CPU */
+    mutex_t xscom_mutex;
 };
 
 /** @fn getCpuId
diff --git a/src/include/sys/mmio.h b/src/include/sys/mmio.h
index eb146da72..c36a13821 100644
--- a/src/include/sys/mmio.h
+++ b/src/include/sys/mmio.h
@@ -2,6 +2,7 @@
 #define __SYS_MMIO_H
 
 #include <stdint.h>
+#include <sys/mutex.h>
 
 #ifdef __cplusplus
 extern "C"
@@ -25,6 +26,20 @@ uint64_t mmio_hmer_read();
  */
 int mmio_hmer_write(uint64_t value);
 
+/** @fn mmio_xscom_mutex()
+ *  @brief Returns the per-CPU mutex for the XSCOM hardware logic.
+ *
+ *  @pre Task must be pinned to a CPU by task_affinity_pin.  Function will
+ *       assert if not met.
+ *
+ *  @returns The existing mutex for the CPU or creates a new one.
+ *
+ *  @note The task should only interact with the mutex while protected by an
+ *        affinity pin.  If the pin is moved the mutex is no longer
+ *        guarenteed for the CPU the task is executing on.
+ */
+mutex_t mmio_xscom_mutex();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index 9f8e1feec..844f8c44a 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -58,6 +58,7 @@ void CpuManager::startCPU(ssize_t i)
         cpu->scheduler_extra = NULL;
 	cpu->kernel_stack = 
 	    (void*) (((uint64_t)PageManager::allocatePage(4)) + 16320);
+        cpu->xscom_mutex = NULL;
 	
 	// Create idle task.
 	cpu->idle_task = TaskManager::createIdleTask();
diff --git a/src/lib/syscall_mmio.C b/src/lib/syscall_mmio.C
index 30ec5bd37..83498798b 100644
--- a/src/lib/syscall_mmio.C
+++ b/src/lib/syscall_mmio.C
@@ -1,5 +1,8 @@
 #include <sys/syscall.h>
 #include <sys/mmio.h>
+#include <assert.h>
+#include <kernel/task.H>
+#include <kernel/cpu.H>
 
 using namespace Systemcalls;
 
@@ -22,3 +25,35 @@ int mmio_hmer_write(uint64_t value)
 {
     return (int)(int64_t)_syscall1(MMIO_HMER_WRITE, (void*)value);
 }
+
+mutex_t mmio_xscom_mutex()
+{
+    // Get task structure.
+    register task_t* task;
+    asm volatile("mr %0, 13" : "=r"(task));
+
+    // Ensure task is pinned.
+    assert(task->affinity_pinned);
+
+    // Get mutex from cpu structure.
+    mutex_t mutex = task->cpu->xscom_mutex;
+
+    // Create mutex if not created.
+    if (NULL == mutex)
+    {
+        mutex = mutex_create();
+
+        // Atomically update xscom_mutex with new mutex.
+        if (!__sync_bool_compare_and_swap(&task->cpu->xscom_mutex, NULL, mutex))
+        {
+            // Failed, some other thread beat us to it.
+
+            // Destroy mutex and get one created by other thread in the
+            // meantime.
+            mutex_destroy(mutex);
+            mutex = task->cpu->xscom_mutex;
+        }
+    }
+
+    return mutex;
+}