Support for master winkle.

RTC: 44730
Change-Id: Ifaeecc659e1bfd8ded4744dc591fc993471519ba
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1471
Tested-by: Jenkins Server
Reviewed-by: Mark W. Wenning <wenning@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

6 files changed, 316 insertions, 41 deletions

diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index e395b2920..2efa89df2 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/include/kernel/cpu.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2010 - 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/include/kernel/cpu.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2010-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 /** @file cpu.H
  *  @brief Defines kernel information and functions about CPUs.
  *
@@ -42,7 +43,7 @@ class Scheduler;
  *  @brief Stores per-CPU kernel information.
  *
  *  @note kernel_stack and master need to be at fixed locations in this
- *        structure due to usages in start.S.
+ *        structure due to usages in start.S (see ppcconsts.S).
  *
  *  - kernel_stack is a double-word at cpu_t[0 bytes].
  *  - master is a byte at cpu_t[12 bytes].
@@ -63,8 +64,11 @@ struct cpu_t
         /** If the CPU is active */
         bool active:1;
 
+        /** If the CPU is winkled */
+        bool winkled:1;
+
         /** Ensure alignment of master attribute for asm code. */
-        uint64_t __reserved_master:30;
+        uint64_t __reserved_master:29;
     } PACKED;
 
     /** Pointer to the scheduler for this CPU (may not be unique) */
@@ -86,6 +90,9 @@ struct cpu_t
 
     /** counter for executePeriodics */
     size_t periodic_count;
+
+    /** Sequence ID of CPU initialization. */
+    uint64_t cpu_start_seqid;
 };
 
 /** @fn getCpuId
diff --git a/src/include/kernel/cpumgr.H b/src/include/kernel/cpumgr.H
index 8cf6d0fc3..fc7b68547 100644
--- a/src/include/kernel/cpumgr.H
+++ b/src/include/kernel/cpumgr.H
@@ -82,10 +82,39 @@ class CpuManager
          */
         static void activateCPU(cpu_t * i_cpu);
 
+        /** @fn deactivateCPU
+         *  Deactivate a cpu to show it is not running.
+         *  @param[in] cpu_t the CPU
+         *  @post Active flag is off in cpu_t structure.
+         */
+        static void deactivateCPU(cpu_t * i_cpu);
+
         /** @fn getCpuCount
-         * Return the number of active CPUs.
+         *  Return the number of active CPUs.
+         */
+        static size_t getCpuCount() { return cv_cpuSeq & 0xFFFFFFFF; }
+
+        /** @fn getCpuSeqId
+         *  Return the current CPU start sequence ID.
+         */
+        static size_t getCpuSeqId() { return cv_cpuSeq >> 32; }
+
+        /** @fn getCpuCountAndSeqId
+         *  Atomically retrieve both the current active CPU count and
+         *  sequence ID.
+         *
          */
-        static size_t getCpuCount() { return cv_cpuCount; }
+        static void getCpuCountAndSeqId(uint32_t& o_count, uint32_t& o_seq)
+        {
+            uint64_t cpuSeq = cv_cpuSeq;
+            o_count = cpuSeq & 0xFFFFFFFF;
+            o_seq = cpuSeq >> 32;
+        }
+
+        /** @fn getThreadCount
+         *  Return the number of HW threads for this CPU core type.
+         */
+        static size_t getThreadCount();
 
         /** @fn startCore
          *  Create structures to support a core activating and start the core.
@@ -127,16 +156,16 @@ class CpuManager
         void startCPU(ssize_t i = -1);
         void startSlaveCPU(cpu_t*);
 
-        static size_t getThreadCount();
-
     private:
         static cpu_t** cv_cpus; // Need to be able to access this
                                         // from start.S to get initial stacks
                                         // of secondary cpus / threads.
 
-        static Barrier cv_barrier;   //!<  barrier for cpus
-        static bool cv_defrag;       //!<  mem heap defrag
-        static size_t cv_cpuCount;   //!<  # of active CPUs
+            /** Number of active CPUs.
+             *  Stored as "(seqID << 32) | cpus"
+             */
+        static uint64_t cv_cpuSeq;
+
         static bool cv_forcedMemPeriodic;       //!<  force free / coalesce.
 
         // If a shutdown of all CPUs is requested
diff --git a/src/include/kernel/deferred.H b/src/include/kernel/deferred.H
new file mode 100644
index 000000000..6d8950acb
--- /dev/null
+++ b/src/include/kernel/deferred.H
@@ -0,0 +1,201 @@
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/include/kernel/deferred.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
+#ifndef __KERNEL_DEFERRED_H
+#define __KERNEL_DEFERRED_H
+
+/** @file deferred.H
+ *  @brief Definition of kernel's deferred work constructs.
+ *
+ *  These classes allow the kernel to synchronize all CPUs into kernel-mode
+ *  and execute a task.  These work items are queued up to be executed at
+ *  the next time-slice expiration (decrementer exception).
+ */
+
+#include <kernel/types.h>
+#include <kernel/spinlock.H>
+#include <kernel/barrier.H>
+
+/** @class DeferredWork
+ *  @brief The base work-item type for deferred work.
+ *
+ *  This class implements the synchronization and state transition for a
+ *  general deferred work object.  This class has methods for defining
+ *  the behavior at specific points in its execution.  A real behavior
+ *  implementation should be defined by inheriting this class and overloading
+ *  the desired virtual methods.
+ *
+ *  The class is closely associated with the DeferredQueue, which is meant to
+ *  hold pending DeferredWork objects.  When a DeferredWork is complete it
+ *  will remove itself from the DeferredQueue and self-delete.
+ *
+ *  The state transition is as follows:
+ *
+ *  <pre>
+ *        MASTER           Both            NON-MASTER
+ *  1.                   Sync all
+ *  2.  masterPreWork                         block
+ *  3.                activeMainWork
+ *  4.                   Sync all
+ *  5.  masterPostWork                        block
+ *  6.              Last thread out deletes
+ *  </pre>
+ *
+ *  With the deferred work there is concern as to what to do with CPUs that
+ *  come online while a deferred work item is being executed.  For instance,
+ *  the Heap Coalescing work could cause data integrity issues if a freshly
+ *  activated CPU were to not block with the rest of the CPUs and instead
+ *  started executed user-space code.  To deal with this condition a
+ *  CPU which is activated after the DeferredWork object was created will
+ *  block at masterPreWork and masterPostWork stages and call
+ *  nonactiveMainWork.  Typically, the non-active CPUs perform no action
+ *  but an implementation could assert for a race-condition that cannot
+ *  be handled properly (though should never occur).
+ */
+class DeferredWork
+{
+    public:
+        /** Default Constructor
+         *  Initializes the work item and determines the CPU-activation
+         *  sequence number at the time of creation.
+         */
+        DeferredWork();
+
+        /** Default destructor
+         *  Verifies that the work item has completed properly and is
+         *  detached from the DeferredQueue.
+         */
+        virtual ~DeferredWork();
+
+        /** Entry point for DeferredWork execution. */
+        void start();
+
+    protected:
+        // Functions to override for desired behavior:
+            /** Executed by master before "main" but after all active
+             *  CPUs are sync'd.
+             */
+        virtual void masterPreWork() {};
+            /** Executed by master after "main" work is complete. */
+        virtual void masterPostWork() {};
+            /** Executed by all [active] CPUs as the "main" work. */
+        virtual void activeMainWork() {};
+            /** Executed by all non-active CPUs as the "main" work. */
+        virtual void nonactiveMainWork() {};
+
+    private:
+        friend class DeferredQueue;
+
+            /** Barrier for synchronization of active CPUs. */
+        Barrier iv_barrier;
+            /** Count of CPUs in object and pointer to next object.
+             *
+             *  Stored as "(cpu count << 32) | (DeferredWork* next)".  This
+             *  is done so that we can atomically check and/or set both
+             *  values.
+             */
+        volatile uint64_t iv_cpus_and_next;
+            /** Sequence ID at the time of creation. */
+        uint32_t iv_activeSeqId;
+            /** Boolean to indicate masterPreWork is complete. */
+        volatile bool iv_releasePre;
+            /** Boolean to indicate masterPostWork is complete. */
+        volatile bool iv_releasePost;
+
+            // Internal state transition functions.
+        void _waitForCpus();
+        void _masterPre();
+        void _waitAtPre();
+        void _masterPost();
+        void _waitAtPost();
+        void _cleanup();
+
+};
+
+/** @class DeferredQueue
+ *  @brief Queue for DeferredWork objects.
+ *
+ *  This class should have a Singleton instance for the whole kernel.
+ *  Periodically, kernel threads can call the 'execute' function to execute
+ *  pending work, if any is available.  Additional work can be 'insert'ed.
+ */
+class DeferredQueue
+{
+    public:
+        /** Default constructor. */
+        DeferredQueue();
+        /** Default destructor. */
+        ~DeferredQueue();
+
+        /** Insert a work item into the queue.
+         *
+         *  @param i_work - The item to add.
+         *
+         *  Ownership of the work item is transfered to the queue, which is
+         *  responsible for delete once the work is complete.
+         */
+        static void insert(DeferredWork* i_work);
+        /** Execute any pending work items. */
+        static void execute();
+
+    private:
+        friend class DeferredWork;
+
+            // Instance functions for static pair.
+        void _insert(DeferredWork* i_work);
+        void _execute();
+
+        /** Remove a completed work item from the queue.
+         *
+         *  This function is to be called by the master CPU executing the
+         *  work item being completed.
+         *
+         *  The work item is atomically removed from the queue and the
+         *  count of CPUs with a pointer to the work item is compared
+         *  against the count of CPUs in the work item.  This function
+         *  returns when all CPUs that have a pointer are actually inside
+         *  the work item (so we can accurately determine when there is
+         *  only a single CPU in the work item for delete).
+         *
+         *  @param i_work - The item to remove.
+         */
+        void _complete(DeferredWork* i_work);
+
+            /** Spinlock to protect insert / remove operations.
+             *
+             *  This is effectively a "write" lock on the structure and all
+             *  reads are done atomically.  The writers have also use atomic
+             *  update operations to ensure the readers (which are never
+             *  locked) operate in a thread-safe manner.
+             */
+        Spinlock lock;
+            /** Count of CPUs in first object and pointer to it.
+             *
+             *  Stored as "(cpu count << 32) | (DeferredWork* ptr)".  This
+             *  is done so that we can atomically check and/or set both
+             *  values.
+             */
+        volatile uint64_t iv_cpus_and_next;
+};
+
+#endif
diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H
index ebec9cc48..c970c337e 100644
--- a/src/include/kernel/misc.H
+++ b/src/include/kernel/misc.H
@@ -30,6 +30,8 @@
 
 #include <stdint.h>
 #include <usr/vmmconst.h>
+#include <kernel/deferred.H>
+#include <kernel/task.H>
 
 namespace KernelMisc
 {
@@ -59,5 +61,32 @@ namespace KernelMisc
         }
         return true;
     }
+
+    /** @class WinkleCore
+     *
+     *  @brief DeferredWork class for implementing cpu_master_winkle.
+     */
+    class WinkleCore : public ::DeferredWork
+    {
+        public:
+            /** ctor
+             *
+             *  @param i_caller - Task calling cpu_master_winkle to be
+             *                    restored after winkle is complete.
+             */
+            WinkleCore(task_t* i_caller) :
+                iv_caller(i_caller), iv_timebase(0) {};
+
+            void masterPreWork();
+            void activeMainWork();
+            void masterPostWork();
+            void nonactiveMainWork();
+
+        private:
+                /** Caller of cpu_master_winkle() */
+            task_t* iv_caller;
+                /** Current time-base value for restore after winkle. */
+            uint64_t iv_timebase;
+    };
 };
 #endif
diff --git a/src/include/kernel/ppcconsts.S b/src/include/kernel/ppcconsts.S
index 1568bb911..8b84e9135 100644
--- a/src/include/kernel/ppcconsts.S
+++ b/src/include/kernel/ppcconsts.S
@@ -1,24 +1,24 @@
 #   IBM_PROLOG_BEGIN_TAG
 #   This is an automatically generated prolog.
-# 
+#
 #   $Source: src/include/kernel/ppcconsts.S $
-# 
+#
 #   IBM CONFIDENTIAL
-# 
+#
 #   COPYRIGHT International Business Machines Corp. 2010-2012
-# 
+#
 #   p1
-# 
+#
 #   Object Code Only (OCO) source materials
 #   Licensed Internal Code Source Materials
 #   IBM HostBoot Licensed Internal Code
-# 
+#
 #   The source code for this program is not published or other-
 #   wise divested of its trade secrets, irrespective of what has
 #   been deposited with the U.S. Copyright Office.
-# 
+#
 #   Origin: 30
-# 
+#
 #   IBM_PROLOG_END_TAG
 #*--------------------------------------------------------------------*#
 #*  Register Constants                                                *#
@@ -243,3 +243,10 @@
 .set TASK_FPR_31, TASK_FPR_30+8
 .set TASK_FPSCR, TASK_FPR_31+8
 
+.set CPU_KERNEL_STACK, 0
+.set CPU_CPUID, CPU_KERNEL_STACK+8
+.set CPU_STATUS, CPU_CPUID+4
+
+.set CPU_STATUS_MASTER, 0
+.set CPU_STATUS_ACTIVE, CPU_STATUS_MASTER+1
+.set CPU_STATUS_WINKLED, CPU_STATUS_ACTIVE+1
diff --git a/src/include/kernel/syscalls.H b/src/include/kernel/syscalls.H
index 5523fc158..b02fd2216 100644
--- a/src/include/kernel/syscalls.H
+++ b/src/include/kernel/syscalls.H
@@ -92,8 +92,10 @@ namespace Systemcalls
         MISC_CPUSTARTCORE,
             /** cpu_spr_value() */
         MISC_CPUSPRVALUE,
-            /** cpu_doze() - Hidden syscall */
-        MISC_CPUDOZE,
+            /** cpu_nap() - Hidden syscall */
+        MISC_CPUNAP,
+            /** cpu_master_winkle() */
+        MISC_CPUWINKLE,
 
             /** mm_alloc_block() */
         MM_ALLOC_BLOCK,