Fixes for winkle state.

    The memory profiling tools sometimes encountered a condition where
    the kernel stack was becoming corrupted.  I tracked it down to the
    winkle code storing the winkle-save state at the wrong end of the
    stack.  Moving the winkle-save area to the bottom of the stack,
    which is where I originally intended it to go.

    Also noticed that the task issuing the winkle was in "running"
    state while waiting for the cores to come out of winkle.  Ensure
    that the kernel updates the task state with a non-running status
    while we are waiting for winkle to complete.

Change-Id: I07a56ea6f24cbc09362f9227d81915da5bc9f148
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1737
Tested-by: Jenkins Server
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

6 files changed, 44 insertions, 28 deletions

diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index 21e4dae74..824a65370 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -53,6 +53,9 @@ struct cpu_t
     /** Stack to use while in kernel mode. */
     void* kernel_stack;
 
+    /** Bottom of the kernel stack. */
+    void* kernel_stack_bottom;
+
     /** ID of the CPU (PIR value) */
     cpuid_t cpu;
 
@@ -93,6 +96,7 @@ struct cpu_t
 
     /** Sequence ID of CPU initialization. */
     uint64_t cpu_start_seqid;
+
 };
 
 /** @fn getCpuId
diff --git a/src/include/kernel/ppcconsts.S b/src/include/kernel/ppcconsts.S
index d888e497e..f76d8cf6b 100644
--- a/src/include/kernel/ppcconsts.S
+++ b/src/include/kernel/ppcconsts.S
@@ -1,25 +1,25 @@
-# IBM_PROLOG_BEGIN_TAG 
-# This is an automatically generated prolog. 
-#  
-# $Source: src/include/kernel/ppcconsts.S $ 
-#  
-# IBM CONFIDENTIAL 
-#  
-# COPYRIGHT International Business Machines Corp. 2010,2012 
-#  
-# p1 
-#  
-# Object Code Only (OCO) source materials 
-# Licensed Internal Code Source Materials 
-# IBM HostBoot Licensed Internal Code 
-#  
-# The source code for this program is not published or otherwise 
-# divested of its trade secrets, irrespective of what has been 
-# deposited with the U.S. Copyright Office. 
-#  
-# Origin: 30 
-#  
-# IBM_PROLOG_END_TAG 
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/include/kernel/ppcconsts.S $
+#
+# IBM CONFIDENTIAL
+#
+# COPYRIGHT International Business Machines Corp. 2010,2012
+#
+# p1
+#
+# Object Code Only (OCO) source materials
+# Licensed Internal Code Source Materials
+# IBM HostBoot Licensed Internal Code
+#
+# The source code for this program is not published or otherwise
+# divested of its trade secrets, irrespective of what has been
+# deposited with the U.S. Copyright Office.
+#
+# Origin: 30
+#
+# IBM_PROLOG_END_TAG
 #*--------------------------------------------------------------------*#
 #*  Register Constants                                                *#
 #*--------------------------------------------------------------------*#
@@ -246,7 +246,8 @@
 .set TASK_FPSCR, TASK_FPR_31+8
 
 .set CPU_KERNEL_STACK, 0
-.set CPU_CPUID, CPU_KERNEL_STACK+8
+.set CPU_KERNEL_STACK_BOTTOM, CPU_KERNEL_STACK+8
+.set CPU_CPUID, CPU_KERNEL_STACK_BOTTOM+8
 .set CPU_STATUS, CPU_CPUID+4
 
 .set CPU_STATUS_MASTER, 0
diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index 791d14703..db79a15bf 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -177,8 +177,15 @@ void CpuManager::startCPU(ssize_t i)
         }
         cpu->scheduler = &Singleton<Scheduler>::instance();
         cpu->scheduler_extra = NULL;
-        cpu->kernel_stack =
-            (void*) (((uint64_t)PageManager::allocatePage(4)) + 16320);
+
+        const size_t kernel_page_count = 4;
+        const size_t kernel_page_offset = kernel_page_count * PAGESIZE -
+                                          8 * sizeof(uint64_t);
+        cpu->kernel_stack_bottom = PageManager::allocatePage(kernel_page_count);
+        cpu->kernel_stack = reinterpret_cast<void*>(
+            reinterpret_cast<uintptr_t>(cpu->kernel_stack_bottom) +
+            kernel_page_offset);
+
         cpu->xscom_mutex = (mutex_t)MUTEX_INITIALIZER;
 
         // Create idle task.
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index 3754003d0..9b2872f04 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -162,7 +162,7 @@ namespace KernelMisc
         task_t* saveArea = new task_t;
         memset(saveArea, '\0', sizeof(task_t));
         saveArea->context.msr_mask = 0xC030; // EE, PR, IR, DR.
-        *(reinterpret_cast<task_t**>(cpu->kernel_stack)) = saveArea;
+        *(reinterpret_cast<task_t**>(cpu->kernel_stack_bottom)) = saveArea;
 
         // Execute winkle.
         kernel_execute_winkle(saveArea);
@@ -190,6 +190,7 @@ namespace KernelMisc
         setTB(iv_timebase);
 
         // Restore caller of cpu_master_winkle().
+        iv_caller->state = TASK_STATE_RUNNING;
         TaskManager::setCurrentTask(iv_caller);
 
     }
diff --git a/src/kernel/start.S b/src/kernel/start.S
index 048c2411e..252014e6a 100644
--- a/src/kernel/start.S
+++ b/src/kernel/start.S
@@ -618,7 +618,7 @@ intvect_system_reset_inactive:
     beq+ _start
 
     ;// Now we are a winkled processor that is awoken.
-    ld r1, CPU_KERNEL_STACK(r2)
+    ld r1, CPU_KERNEL_STACK_BOTTOM(r2)
     ld r1, 0(r1)
     mtsprg3 r1
     b kernel_dispatch_task
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index a38f08b78..4eb3b4fa8 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -701,7 +701,10 @@ namespace Systemcalls
         else
         {
             TASK_SETRTN(t, 0);
-            DeferredQueue::insert(new KernelMisc::WinkleCore(t));
+            KernelMisc::WinkleCore* deferred = new KernelMisc::WinkleCore(t);
+            t->state = TASK_STATE_BLOCK_USRSPACE;
+            t->state_info = deferred;
+            DeferredQueue::insert(deferred);
             TaskManager::setCurrentTask(cpu->idle_task);
             DeferredQueue::execute();
         }