Kernel support for processor affinity.

Change-Id: Ie84a805bad58032085208a98b1b31393def681cb
Reviewed-on: http://gfwr801.rchland.ibm.com:8080/gerrit/100
Tested-by: Jenkins Server
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: Andrew J. Geissler <andrewg@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>

6 files changed, 82 insertions, 8 deletions

diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index 4d5134114..0fb794989 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -1,3 +1,8 @@
+/** @file cpu.H
+ *  @brief Defines kernel information and functions about CPUs.
+ *  
+ *  In this kernel the term CPU refers to a hardware thread (SMT), not core.
+ */
 #ifndef __KERNEL_CPU_H
 #define __KERNEL_CPU_H
 
@@ -10,17 +15,31 @@
 
 class Scheduler;
 
+/** @struct cpu_t
+ *  @brief Stores per-CPU kernel information.
+ */
 struct cpu_t
 {
+        /** Stack to use while in kernel mode. */
     void* kernel_stack;
+        /** ID of the CPU (PIR value) */
     cpuid_t cpu;
-
+        
+        /** Pointer to the scheduler for this CPU (may not be unique) */
     Scheduler* scheduler;
+        /** Location for scheduler to store per-CPU data, currently used
+         *  for the local run-queue for processor affinity.
+         */
+    void* scheduler_extra;
+        /** Pointer to the idle task for this CPU */
     task_t* idle_task;
 };
 
+/** @fn getCpuId
+ *  @brief Read the PIR value to determine the cpuid_t of this CPU.
+ */
 ALWAYS_INLINE
-inline uint64_t getCpuId()
+inline cpuid_t getCpuId()
 {
     return getPIR() & (KERNEL_MAX_SUPPORTED_CPUS - 1);
 }
diff --git a/src/include/kernel/scheduler.H b/src/include/kernel/scheduler.H
index 71b21559a..9cbf83d39 100644
--- a/src/include/kernel/scheduler.H
+++ b/src/include/kernel/scheduler.H
@@ -23,7 +23,8 @@ class Scheduler
 	~Scheduler() {};
 
     private:
-	Util::Locked::Queue<task_t, true, Spinlock> iv_taskList;
+        typedef Util::Locked::Queue<task_t, true, Spinlock> Runqueue_t;
+        Runqueue_t iv_taskList;
 };
 
 #endif
diff --git a/src/include/kernel/task.H b/src/include/kernel/task.H
index c27c7e8ce..6cd49028d 100644
--- a/src/include/kernel/task.H
+++ b/src/include/kernel/task.H
@@ -1,8 +1,16 @@
+/** @file task.H
+ *  @brief Defines kernel information about tasks.
+ */
 #ifndef __KERNEL_TASK_H
 #define __KERNEL_TASK_H
 
 #include <kernel/types.h>
 
+/** @struct context_t
+ *  @brief Defines the save-restore context for the task.
+ *
+ *  See PowerISA for information on registers listed.
+ */
 struct context_t
 {
     void* stack_ptr;
@@ -14,12 +22,25 @@ struct context_t
     uint64_t xer;
 };
 
+/** @struct task_t
+ *  @brief The kernel-level task structure.
+ */
 struct task_t
 {
+        /** Pointer to the CPU this task is assigned to. */
     cpu_t* cpu;
+        /** Context information.  This MUST stay here due to 
+         *  save-restore asm code. */
     context_t context;
-    
+        
+        /** Task ID */
     tid_t tid;
+        /** Determines if user-space would like this task pinned to a CPU.
+         *  This value is considered a count of the number of times the pinned
+         *  as been requested, so pinning can be used recursively. */
+    uint64_t affinity_pinned;
+        
+        // Pointers for queue containers.
     task_t* prev;
     task_t* next;
 };
diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index 46d1baa93..9f8e1feec 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -55,6 +55,7 @@ void CpuManager::startCPU(ssize_t i)
 	// Initialize CPU.
 	cpu->cpu = i;
 	cpu->scheduler = &Singleton<Scheduler>::instance();
+        cpu->scheduler_extra = NULL;
 	cpu->kernel_stack = 
 	    (void*) (((uint64_t)PageManager::allocatePage(4)) + 16320);
 	
diff --git a/src/kernel/scheduler.C b/src/kernel/scheduler.C
index cb3cb5e95..ef0c3c005 100644
--- a/src/kernel/scheduler.C
+++ b/src/kernel/scheduler.C
@@ -10,7 +10,22 @@ void Scheduler::addTask(task_t* t)
 {
     if (t->cpu->idle_task != t)
     {
-	iv_taskList.insert(t);
+        // If task is pinned to this CPU, add to the per-CPU queue.
+        if (0 != t->affinity_pinned)
+        {
+            // Allocate a per-CPU queue if this is the first pinning CPU.
+            if (NULL == t->cpu->scheduler_extra)
+            {
+                t->cpu->scheduler_extra = new Runqueue_t();
+            }
+            // Insert into queue.
+            static_cast<Runqueue_t*>(t->cpu->scheduler_extra)->insert(t);
+        }
+        // Not pinned, add to global run-queue.
+        else
+        {
+	    iv_taskList.insert(t);
+        }
     }
 }
 
@@ -21,11 +36,25 @@ void Scheduler::returnRunnable()
 
 void Scheduler::setNextRunnable()
 {
-    task_t* t = iv_taskList.remove();
+    task_t* t = NULL;
+    cpu_t* cpu = CpuManager::getCurrentCPU();
+    
+    // Check for ready task in local run-queue, if it exists.
+    if (NULL != cpu->scheduler_extra)
+    {
+        t = static_cast<Runqueue_t*>(cpu->scheduler_extra)->remove();
+    }
+    
+    // Check for ready task in global run-queue.
+    if (NULL == t)
+    {
+        t = iv_taskList.remove();
+    }
 
+    // Choose idle task if no other ready task is available.
     if (NULL == t)
     {
-	t = CpuManager::getCurrentCPU()->idle_task;
+	t = cpu->idle_task;
 	// TODO: Set short decrementer.
         setDEC(TimeManager::getTimeSliceCount());
     }
@@ -33,6 +62,6 @@ void Scheduler::setNextRunnable()
     {
         setDEC(TimeManager::getTimeSliceCount());
     }
-    
+   
     TaskManager::setCurrentTask(t);
 }
diff --git a/src/kernel/taskmgr.C b/src/kernel/taskmgr.C
index 0ddc3918a..8225b7da3 100644
--- a/src/kernel/taskmgr.C
+++ b/src/kernel/taskmgr.C
@@ -4,6 +4,7 @@
 #include <kernel/pagemgr.H>
 #include <kernel/cpumgr.H>
 #include <arch/ppc.H>
+#include <string.h>
 
 void TaskManager::idleTaskLoop(void* unused)
 {
@@ -49,6 +50,8 @@ task_t* TaskManager::_createTask(TaskManager::task_fn_t t,
 				 void* p, bool withStack)
 {
     task_t* task = new task_t;
+    memset(task, '\0', sizeof(task_t));
+
     task->tid = this->getNextTid();
 
     // Function pointer 't' is actually a TOC entry.