Support for core_activate via IPI.

RTC: 37009
Change-Id: I56669805c86d9659a20ad7c26e5e9860c7a248c7
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1087
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

23 files changed, 396 insertions, 160 deletions

diff --git a/src/build/simics/hb-simdebug.py b/src/build/simics/hb-simdebug.py
index c8aba37e1..70d9190d4 100755
--- a/src/build/simics/hb-simdebug.py
+++ b/src/build/simics/hb-simdebug.py
@@ -192,8 +192,10 @@ Examples: \n
 #------------------------------------------------
 #------------------------------------------------
 def hb_singlethread():
-    run_command("foreach $cpu in (system_cmp0.get-processor-list) {$cpu.disable}")
-    run_command("cpu0_0_0_0.enable");
+    run_command("foreach $cpu in (system_cmp0.get-processor-list) " +
+                "{ pdisable $cpu}");
+    run_command("penable cpu0_0_05_0");
+    run_command("pselect cpu0_0_05_0");
     return
 
 new_command("hb-singlethread",
@@ -201,4 +203,4 @@ new_command("hb-singlethread",
     [],
     alias = "hb-st",
     type = ["hostboot-commands"],
-    short = "Disable all threads except cpu0_0_0_0.")
+    short = "Disable all threads except cpu0_0_05_0.")
diff --git a/src/include/arch/ppc.H b/src/include/arch/ppc.H
index 5d2e51a7b..7807b5743 100644
--- a/src/include/arch/ppc.H
+++ b/src/include/arch/ppc.H
@@ -324,6 +324,12 @@ inline void MAGIC_INSTRUCTION(int _n)
 
 enum
 {
+    MAGIC_SIMICS_CORESTATESAVE = 10, // Indicate to the PHYP model of simics
+                                     // that we are preparing to wake up a core
+                                     // or thread.  This allows them to save
+                                     // some state from the core doing the
+                                     // wakeup to apply into the woken one.
+
     MAGIC_SHUTDOWN            =  7006, // KernelMisc::shutdown() called.
     MAGIC_BREAK               =  7007, // hard-code a breakpoint
     MAGIC_CONTINUOUS_TRACE    =  7055, // extract mixed trace buffer
diff --git a/src/include/kernel/cpu.H b/src/include/kernel/cpu.H
index b6fe66ef2..e395b2920 100644
--- a/src/include/kernel/cpu.H
+++ b/src/include/kernel/cpu.H
@@ -75,6 +75,9 @@ struct cpu_t
      */
     void* scheduler_extra;
 
+    /** Location for task delay-list, managed by TimeManager. */
+    void* delay_list;
+
     /** Pointer to the idle task for this CPU */
     task_t* idle_task;
 
diff --git a/src/include/kernel/cpumgr.H b/src/include/kernel/cpumgr.H
index e4d2b8b0b..d7bc2da78 100644
--- a/src/include/kernel/cpumgr.H
+++ b/src/include/kernel/cpumgr.H
@@ -36,7 +36,7 @@ class CpuManager
         {
             MAXCPUS = KERNEL_MAX_SUPPORTED_CPUS,
             CPU_PERIODIC_CHECK_MEMORY = 128,    // Is this even needed anymore?
-            CPU_PERIODIC_FLUSH_PAGETABLE = 256, 
+            CPU_PERIODIC_FLUSH_PAGETABLE = 256,
             CPU_PERIODIC_DEFRAG = 949,  // TODO Any bigger not currently hit
         };
 
@@ -44,7 +44,7 @@ class CpuManager
          *  Returns a pointer to the current CPU structure by using the
          *  task structure in SPRG3.
          */
-        static cpu_t* getCurrentCPU();
+        static cpu_t* getCurrentCPU() { return cv_cpus[getPIR()]; }
         static cpu_t* getCpu(size_t i) { return cv_cpus[i]; }
 
         /** @brief Return pointer to master CPU object.
@@ -87,6 +87,13 @@ class CpuManager
          */
         static size_t getCpuCount() { return cv_cpuCount; }
 
+        /** @fn startCore
+         *  Create structures to support a core activating and start the core.
+         *
+         *  @param[in] pir - PIR value of first thread in core.
+         */
+        static int startCore(uint64_t pir);
+
 
         /** @fn forceMemoryPeriodic()
          * Force the memory free / coalesce operations to be performed on the
@@ -105,8 +112,10 @@ class CpuManager
         void startCPU(ssize_t i = -1);
         void startSlaveCPU(cpu_t*);
 
+        static size_t getThreadCount();
+
     private:
-        static cpu_t* cv_cpus[MAXCPUS]; // Need to be able to access this
+        static cpu_t** cv_cpus; // Need to be able to access this
                                         // from start.S to get initial stacks
                                         // of secondary cpus / threads.
 
diff --git a/src/include/kernel/syscalls.H b/src/include/kernel/syscalls.H
index a46fde703..0a0d5dba5 100644
--- a/src/include/kernel/syscalls.H
+++ b/src/include/kernel/syscalls.H
@@ -88,6 +88,8 @@ namespace Systemcalls
         MISC_CPUCORETYPE,
             /** cpu_dd_level() */
         MISC_CPUDDLEVEL,
+            /** cpu_start_core() */
+        MISC_CPUSTARTCORE,
 
             /** mm_alloc_block() */
         MM_ALLOC_BLOCK,
diff --git a/src/include/kernel/timemgr.H b/src/include/kernel/timemgr.H
index 0590dedff..96950b928 100644
--- a/src/include/kernel/timemgr.H
+++ b/src/include/kernel/timemgr.H
@@ -47,28 +47,32 @@ struct _TimeManager_Delay_t
 class TimeManager
 {
     public:
-	enum
-	    {
+        enum
+            {
                 /** Number of time-slices to allow per second.
                  *
                  *  Context length becomes (1/TIMESLICE_PER_SECOND) sec.
                  */
-		TIMESLICE_PER_SEC = 1000,
-	    };
+                TIMESLICE_PER_SEC = 1000,
+            };
 
             /** Initialize the time subsystem. */
-	static void init();
+        static void init();
+
+            /** Initialize the task-delay structures for a CPU. */
+        static void init_cpu(cpu_t* cpu);
+
             /** Return the number of ticks per time-slice. */
-	static uint64_t getTimeSliceCount()
-	    {
-		return iv_timebaseFreq / TIMESLICE_PER_SEC;
-	    };
+        static uint64_t getTimeSliceCount()
+            {
+                return iv_timebaseFreq / TIMESLICE_PER_SEC;
+            };
 
             /** Returns the value of the processor timebase register. */
-	static uint64_t getCurrentTimeBase()
-	    {
-		return getTB();
-	    };
+        static uint64_t getCurrentTimeBase()
+            {
+                return getTB();
+            };
 
         /** Converts seconds/nsecs to timebase ticks.
          *
@@ -80,7 +84,7 @@ class TimeManager
          *
          *  @return Number of timebase ticks.
          */
-	static uint64_t convertSecToTicks(uint64_t i_sec, uint64_t i_nsec);
+        static uint64_t convertSecToTicks(uint64_t i_sec, uint64_t i_nsec);
         /** Converts timebase ticks to seconds/nsecs.
          *
          *  @param[in] i_ticks - Number of ticks.
@@ -96,23 +100,27 @@ class TimeManager
          *  @param[in] i_sec - Seconds.
          *  @param[in] i_nsec - Nsecs.
          */
-	static void delayTask(task_t* t, uint64_t i_sec, uint64_t i_nsec);
+        static void delayTask(task_t* t, uint64_t i_sec, uint64_t i_nsec);
         /** Checks the sleep queue to determine if any tasks should be woken. */
-	static void checkReleaseTasks(Scheduler* s);
+        static void checkReleaseTasks(Scheduler* s);
 
     protected:
-	TimeManager() {};
-	~TimeManager() {};
+        TimeManager() {};
+        ~TimeManager() {};
 
     private:
-	void _init();
-	void _delayTask(task_t* t, uint64_t i_sec, uint64_t i_nsec);
-	void _checkReleaseTasks(Scheduler* s);
 
-	Util::Locked::PQueue<_TimeManager_Delay_t, uint64_t>
-			iv_taskList[KERNEL_MAX_SUPPORTED_CPUS];
+        typedef Util::Locked::PQueue<_TimeManager_Delay_t, uint64_t>
+            delaylist_t;
+
+        void _init();
+        void _init_cpu(cpu_t* cpu);
+        void _delayTask(task_t* t, uint64_t i_sec, uint64_t i_nsec);
+        void _checkReleaseTasks(Scheduler* s);
+        inline delaylist_t* _get_delaylist();
+
         /** Frequency of the timebase register in Hz. (ticks per second) */
-	static uint64_t iv_timebaseFreq;
+        static uint64_t iv_timebaseFreq;
 };
 
 #endif
diff --git a/src/include/sys/misc.h b/src/include/sys/misc.h
index fcf88f08a..90ff57964 100644
--- a/src/include/sys/misc.h
+++ b/src/include/sys/misc.h
@@ -92,11 +92,26 @@ ProcessorCoreType cpu_core_type();
 uint8_t cpu_dd_level();
 
 /** @fn cpu_thread_count()
- *  @breif Get the number of threads per cpu for this proctype
+ *  @brief Get the number of threads per cpu for this proctype
  *  @return # of threads per cpu
  */
 size_t cpu_thread_count();
 
+/** @fn cpu_start_core
+ *  @brief Have the kernel start a new core.
+ *
+ *  @param[in] pir - PIR value of the first thread on the core.
+ *
+ *  @note The kernel will start all threads on the requested core even
+ *        though the callee only requests with a single PIR value.
+ *
+ *  @return 0 or -(errno) on failure.
+ *
+ *  @retval -ENXIO - The core ID was outside of the range the kernel is
+ *                   prepared to support.
+ */
+int cpu_start_core(uint64_t pir);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/include/sys/vfs.h b/src/include/sys/vfs.h
index 254a10350..b4b7da594 100644
--- a/src/include/sys/vfs.h
+++ b/src/include/sys/vfs.h
@@ -32,7 +32,7 @@
 #endif
 
 // make TODO VFS_MODULE_MAX equal to the actual number of modules in the base image (+ 2?)
-#define VFS_MODULE_MAX 64
+#define VFS_MODULE_MAX 16
 // Extended use 4 4k pages
 //  Extended Module Virtual address at 1GB
 #define VFS_EXTENDED_MODULE_VADDR (1 * 1024 * 1024 * 1024)
@@ -51,7 +51,7 @@
 #define VFS_MODULE_DEFINE_START(f) \
     extern "C" void VFS_SYMBOL_START(void* args) \
     { \
-	f(args); \
+        f(args); \
     }
 
 #ifdef __cplusplus
diff --git a/src/include/usr/hwpf/istepreasoncodes.H b/src/include/usr/hwpf/istepreasoncodes.H
index fc44b2c87..c186e1e87 100644
--- a/src/include/usr/hwpf/istepreasoncodes.H
+++ b/src/include/usr/hwpf/istepreasoncodes.H
@@ -50,6 +50,7 @@ enum istepModuleId
     ISTEP_STARTPAYLOAD_EXECUTE_UNIT_TESTS               = 0x01,
     ISTEP_START_PAYLOAD_CALL_SHUTDOWN                   = 0x02,
     ISTEP_START_PAYLOAD_NOTIFY_FSP                      = 0x03,
+    ISTEP_ACTIVATE_SLAVE_CORES                          = 0x04,
 };
 
 /**
@@ -64,6 +65,7 @@ enum istepReasonCode
     ISTEP_CXXTEST_FAILED_TEST                   = ISTEP_COMP_ID | 0x01,
     ISTEP_TARGET_NULL                           = ISTEP_COMP_ID | 0x02,
     ISTEP_MBOX_MSG_NULL                         = ISTEP_COMP_ID | 0x03,
+    ISTEP_BAD_RC                                = ISTEP_COMP_ID | 0x04,
 };
 
 };  // end ISTEP
diff --git a/src/include/usr/intr/interrupt.H b/src/include/usr/intr/interrupt.H
index 51fe950e0..751cedc07 100644
--- a/src/include/usr/intr/interrupt.H
+++ b/src/include/usr/intr/interrupt.H
@@ -60,11 +60,10 @@ namespace INTR
      */
     enum msg_intr_types_t
     {
-        MSG_INTR_ADD_CPU_USR = 1,  //!< Add cpu core, data[0] = cpuid (PIR)
-        MSG_INTR_REGISTER_MSGQ,    //!< Register a msgQ
-        MSG_INTR_UNREGISTER_MSGQ,  //!< Un register a msgQ
-        MSG_INTR_ENABLE,           //!< Enable external Interrupts
-        MSG_INTR_DISABLE,          //!< Disable external interrupts
+        MSG_INTR_REGISTER_MSGQ = 1, //!< Register a msgQ
+        MSG_INTR_UNREGISTER_MSGQ,   //!< Un register a msgQ
+        MSG_INTR_ENABLE,            //!< Enable external Interrupts
+        MSG_INTR_DISABLE,           //!< Disable external interrupts
         MSG_INTR_SHUTDOWN,         //!< Call to shutdown interrupt presenter
     };
 
diff --git a/src/include/util/impl/stlmap.H b/src/include/util/impl/stlmap.H
index 97379e861..08d293024 100644
--- a/src/include/util/impl/stlmap.H
+++ b/src/include/util/impl/stlmap.H
@@ -531,7 +531,7 @@ namespace Util
                  * This is a fairly straight-forward cast since no address
                  * translation is necessary.
                  */
-                const void** makeSplayTreeValue(const value_type& x)
+                const void** makeSplayTreeValue(const value_type& x) const
                 {
                     return reinterpret_cast<const void**>(
                                 const_cast<value_type*>(&x));
diff --git a/src/kernel/cpumgr.C b/src/kernel/cpumgr.C
index 4fdc8fde9..48bf4a5e8 100644
--- a/src/kernel/cpumgr.C
+++ b/src/kernel/cpumgr.C
@@ -36,8 +36,10 @@
 #include <kernel/cpuid.H>
 #include <kernel/ptmgr.H>
 #include <kernel/heapmgr.H>
+#include <kernel/intmsghandler.H>
+#include <errno.h>
 
-cpu_t* CpuManager::cv_cpus[CpuManager::MAXCPUS] = { NULL };
+cpu_t** CpuManager::cv_cpus = NULL;
 bool CpuManager::cv_shutdown_requested = false;
 uint64_t CpuManager::cv_shutdown_status = 0;
 Barrier CpuManager::cv_barrier;
@@ -49,16 +51,11 @@ InteractiveDebug CpuManager::cv_interactive_debug;
 CpuManager::CpuManager()
 {
     for (int i = 0; i < MAXCPUS; i++)
-	cv_cpus[i] = NULL;
+        cv_cpus[i] = NULL;
 
     memset(&cv_interactive_debug, '\0', sizeof(cv_interactive_debug));
 }
 
-cpu_t* CpuManager::getCurrentCPU()
-{
-    return cv_cpus[getPIR()];
-}
-
 cpu_t* CpuManager::getMasterCPU()
 {
     for (int i = 0; i < MAXCPUS; i++)
@@ -77,27 +74,13 @@ void CpuManager::init()
     // enter the kernel, so we skip initializing all the other CPUs for now.
 
     // Determine number of threads on this core.
-    size_t threads = -1;
-    switch (CpuID::getCpuType())
-    {
-        case CORE_POWER7:
-        case CORE_POWER7_PLUS:
-            threads = 4;
-            break;
-
-        case CORE_POWER8_VENICE:
-        case CORE_POWER8_MURANO:
-            threads = 8;
-            break;
+    size_t threads = getThreadCount();
 
-        case CORE_UNKNOWN:
-        default:
-            kassert(false);
-            break;
-    }
+    // Set up CPU structure.
+    cv_cpus = new cpu_t*[MAXCPUS];
 
     // Create CPU objects starting at the thread-0 for this core.
-    size_t baseCpu = getPIR() & ~(threads-1);
+    size_t baseCpu = getCpuId() & ~(threads-1);
     for (size_t i = 0; i < threads; i++)
         Singleton<CpuManager>::instance().startCPU(i + baseCpu);
 }
@@ -119,22 +102,22 @@ void CpuManager::startCPU(ssize_t i)
     bool currentCPU = false;
     if (i < 0)
     {
-	i = getCpuId();
-	currentCPU = true;
+        i = getCpuId();
+        currentCPU = true;
     }
     else if (getCpuId() == (uint64_t)i)
     {
-	currentCPU = true;
+        currentCPU = true;
     }
 
     // Initialize CPU structure.
     if (NULL == cv_cpus[i])
     {
-	printk("Starting CPU %ld...", i);
-	cpu_t* cpu = cv_cpus[i] = new cpu_t;
+        printk("Starting CPU %ld...", i);
+        cpu_t* cpu = cv_cpus[i] = new cpu_t;
 
-	// Initialize CPU.
-	cpu->cpu = i;
+        // Initialize CPU.
+        cpu->cpu = i;
         if (currentCPU)
         {
             cpu->master = true;
@@ -143,18 +126,21 @@ void CpuManager::startCPU(ssize_t i)
         {
             cpu->master = false;
         }
-	cpu->scheduler = &Singleton<Scheduler>::instance();
+        cpu->scheduler = &Singleton<Scheduler>::instance();
         cpu->scheduler_extra = NULL;
-	cpu->kernel_stack =
-	    (void*) (((uint64_t)PageManager::allocatePage(4)) + 16320);
+        cpu->kernel_stack =
+            (void*) (((uint64_t)PageManager::allocatePage(4)) + 16320);
         cpu->xscom_mutex = (mutex_t)MUTEX_INITIALIZER;
 
-	// Create idle task.
-	cpu->idle_task = TaskManager::createIdleTask();
-	cpu->idle_task->cpu = cpu;
+        // Create idle task.
+        cpu->idle_task = TaskManager::createIdleTask();
+        cpu->idle_task->cpu = cpu;
         cpu->periodic_count = 0;
 
-	printk("done\n");
+        // Call TimeManager setup for a CPU.
+        TimeManager::init_cpu(cpu);
+
+        printk("done\n");
     }
 
     if (currentCPU)
@@ -176,7 +162,7 @@ void CpuManager::startSlaveCPU(cpu_t* cpu)
 void CpuManager::activateCPU(cpu_t * i_cpu)
 {
     i_cpu->active = true;
-    __sync_fetch_and_add(&cv_cpuCount, 1);
+    __sync_add_and_fetch(&cv_cpuCount, 1);
     lwsync();
 }
 
@@ -247,6 +233,49 @@ void CpuManager::executePeriodics(cpu_t * i_cpu)
     }
 }
 
+int CpuManager::startCore(uint64_t pir)
+{
+    size_t threads = getThreadCount();
+    pir = pir & ~(threads-1);
+
+
+    if (pir >= MAXCPUS) { return -ENXIO; }
+
+    for(size_t i = 0; i < threads; i++)
+    {
+        Singleton<CpuManager>::instance().startCPU(pir + i);
+    }
+    __sync_synchronize();
+
+    InterruptMsgHdlr::addCpuCore(pir);
+
+    return 0;
+};
+
+size_t CpuManager::getThreadCount()
+{
+    size_t threads = 0;
+    switch (CpuID::getCpuType())
+    {
+        case CORE_POWER7:
+        case CORE_POWER7_PLUS:
+            threads = 4;
+            break;
+
+        case CORE_POWER8_VENICE:
+        case CORE_POWER8_MURANO:
+            threads = 8;
+            break;
+
+        case CORE_UNKNOWN:
+        default:
+            kassert(false);
+            break;
+    }
+
+    return threads;
+}
+
 void CpuManager::forceMemoryPeriodic()
 {
     cv_forcedMemPeriodic = true;
diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C
index 98f941f4c..73d14e787 100644
--- a/src/kernel/intmsghandler.C
+++ b/src/kernel/intmsghandler.C
@@ -99,8 +99,8 @@ void InterruptMsgHdlr::handleInterrupt()
     if(cv_instance)
     {
         cv_instance->sendMessage(MSG_INTR_EXTERN,
-                                 (void *)xirr,
                                  (void *)pir,
+                                 (void *)xirr,
                                  NULL);
     }
 
@@ -114,13 +114,18 @@ void InterruptMsgHdlr::handleInterrupt()
 
 }
 
-
-// TODO where does this get called from? (story 39878)
 void InterruptMsgHdlr::addCpuCore(uint64_t i_pir)
 {
+    task_t* t = TaskManager::getCurrentTask();
+
     if(cv_instance)
     {
-        cv_instance->sendMessage(MSG_INTR_ADD_CPU,(void *)i_pir,NULL,NULL);
+        // To avoid conflict with interrupts on thread i_pir, change the key
+        // for the message to be an invalid PIR.
+        uint64_t pir_key = i_pir | 0x8000000000000000ul;
+
+        cv_instance->sendMessage(MSG_INTR_ADD_CPU,
+                                 (void*)pir_key,(void *)i_pir,t);
     }
 }
 
diff --git a/src/kernel/msghandler.C b/src/kernel/msghandler.C
index 71a9d84d1..7761d36ae 100644
--- a/src/kernel/msghandler.C
+++ b/src/kernel/msghandler.C
@@ -42,6 +42,13 @@ void MessageHandler::sendMessage(msg_sys_types_t i_type, void* i_key,
     mhp->key = i_key;
     mhp->task = i_task;
 
+    // Update block status for task.
+    if (NULL != i_task)
+    {
+        i_task->state = TASK_STATE_BLOCK_USRSPACE;
+        i_task->state_info = i_key;
+    }
+
     // Send userspace message if one hasn't been sent for this key.
     if (!iv_pending.find(i_key))
     {
@@ -76,10 +83,6 @@ void MessageHandler::sendMessage(msg_sys_types_t i_type, void* i_key,
     // Defer task while waiting for message response.
     if (NULL != i_task)
     {
-        // Set block status.
-        i_task->state = TASK_STATE_BLOCK_USRSPACE;
-        i_task->state_info = i_key;
-
         if (i_task == TaskManager::getCurrentTask())
         {
             // Switch to ready waiter, or pick a new task off the scheduler.
diff --git a/src/kernel/start.S b/src/kernel/start.S
index 6d0eab195..a8166b980 100644
--- a/src/kernel/start.S
+++ b/src/kernel/start.S
@@ -272,6 +272,7 @@ _other_thread_spinlock_complete:
     mfspr r1, PIR
     lis r2, _ZN10CpuManager7cv_cpusE@h
     ori r2, r2, _ZN10CpuManager7cv_cpusE@l
+    ld r2, 0(r2)                ;// Dereference cv_cpus to get array.
     muli r3, r1, 8
     add r2, r3, r2
     ld r3, 0(r2)		;// Load CPU object.
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index b061a4877..26a72cd35 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -96,6 +96,7 @@ namespace Systemcalls
     void Shutdown(task_t *t);
     void CpuCoreType(task_t *t);
     void CpuDDLevel(task_t *t);
+    void CpuStartCore(task_t *t);
     void MmAllocBlock(task_t *t);
     void MmRemovePages(task_t *t);
     void MmSetPermission(task_t *t);
@@ -129,6 +130,7 @@ namespace Systemcalls
         &Shutdown,    // MISC_SHUTDOWN
         &CpuCoreType, // MISC_CPUCORETYPE
         &CpuDDLevel,  // MISC_CPUDDLEVEL
+        &CpuStartCore, // MISC_CPUSTARTCORE
 
         &MmAllocBlock, // MM_ALLOC_BLOCK
         &MmRemovePages, // MM_REMOVE_PAGES
@@ -631,6 +633,13 @@ namespace Systemcalls
         TASK_SETRTN(t, CpuID::getCpuDD());
     }
 
+    /** Prep core for activation. */
+    void CpuStartCore(task_t *t)
+    {
+        TASK_SETRTN(t,
+            CpuManager::startCore(static_cast<uint64_t>(TASK_GETARG0(t))));
+    };
+
     /**
      * Allocate a block of virtual memory within the base segment
      * @param[in] t: The task used to allocate a block in the base segment
diff --git a/src/kernel/timemgr.C b/src/kernel/timemgr.C
index b6be8cb90..36d89ee30 100644
--- a/src/kernel/timemgr.C
+++ b/src/kernel/timemgr.C
@@ -24,6 +24,7 @@
 #include <kernel/scheduler.H>
 #include <util/singleton.H>
 #include <kernel/task.H>
+#include <kernel/cpumgr.H>
 
 uint64_t TimeManager::iv_timebaseFreq = 0xFFFFFFFF;
 
@@ -37,6 +38,16 @@ void TimeManager::_init()
     iv_timebaseFreq = 512000000ULL;
 }
 
+void TimeManager::init_cpu(cpu_t* cpu)
+{
+    Singleton<TimeManager>::instance()._init_cpu(cpu);
+}
+
+void TimeManager::_init_cpu(cpu_t* cpu)
+{
+    cpu->delay_list = new delaylist_t;
+}
+
 uint64_t TimeManager::convertSecToTicks(uint64_t i_sec, uint64_t i_nsec)
 {
     // This code will handle times almost up to a year without overflowing a
@@ -68,13 +79,13 @@ void TimeManager::_delayTask(task_t* t, uint64_t i_sec, uint64_t i_nsec)
     _TimeManager_Delay_t* node = new _TimeManager_Delay_t();
 
     node->key = this->getCurrentTimeBase() +
-		this->convertSecToTicks(i_sec, i_nsec);
+                this->convertSecToTicks(i_sec, i_nsec);
     node->task = t;
 
     t->state = TASK_STATE_BLOCK_SLEEP;
     t->state_info = (void*)node->key;
 
-    iv_taskList[getPIR()].insert(node);
+    _get_delaylist()->insert(node);
 }
 
 void TimeManager::checkReleaseTasks(Scheduler* s)
@@ -87,9 +98,14 @@ void TimeManager::_checkReleaseTasks(Scheduler* s)
     uint64_t l_currentTime = getCurrentTimeBase();
     _TimeManager_Delay_t* node = NULL;
 
-    while(NULL != (node = iv_taskList[getPIR()].remove_if(l_currentTime)))
+    while(NULL != (node = _get_delaylist()->remove_if(l_currentTime)))
     {
-	s->addTask(node->task);
-	delete node;
+        s->addTask(node->task);
+        delete node;
     }
 }
+
+inline TimeManager::delaylist_t* TimeManager::_get_delaylist()
+{
+    return static_cast<delaylist_t*>(CpuManager::getCurrentCPU()->delay_list);
+}
diff --git a/src/lib/syscall_misc.C b/src/lib/syscall_misc.C
index 539399b7a..3124342c1 100644
--- a/src/lib/syscall_misc.C
+++ b/src/lib/syscall_misc.C
@@ -70,3 +70,9 @@ size_t cpu_thread_count()
     return threads;
 }
 
+int cpu_start_core(uint64_t pir)
+{
+    return reinterpret_cast<int64_t>(
+        _syscall1(MISC_CPUSTARTCORE, reinterpret_cast<void*>(pir)));
+}
+
diff --git a/src/usr/hwpf/hwp/core_activate/core_activate.C b/src/usr/hwpf/hwp/core_activate/core_activate.C
index c79e0a6d4..e4ecf4ba1 100644
--- a/src/usr/hwpf/hwp/core_activate/core_activate.C
+++ b/src/usr/hwpf/hwp/core_activate/core_activate.C
@@ -47,12 +47,16 @@
 
 //  targeting support
 #include    <targeting/common/commontargeting.H>
+#include    <targeting/common/utilFilter.H>
 
 //  fapi support
 #include    <fapi.H>
 #include    <fapiPlatHwpInvoker.H>
+#include    <hwpf/istepreasoncodes.H>
 
 #include    "core_activate.H"
+#include    <sys/task.h>
+#include    <sys/misc.h>
 
 //  Uncomment these files as they become available:
 // #include    "host_activate_master/host_activate_master.H"
@@ -64,7 +68,7 @@ namespace   CORE_ACTIVATE
 
 using   namespace   TARGETING;
 using   namespace   fapi;
-
+using   namespace   ISTEP;
 
 
 //
@@ -131,38 +135,71 @@ void    call_host_activate_slave_cores( void    *io_pArgs )
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_slave_cores entry" );
 
-#if 0
     // @@@@@    CUSTOM BLOCK:   @@@@@
-    //  figure out what targets we need
-    //  customize any other inputs
-    //  set up loops to go through all targets (if parallel, spin off a task)
+#if 0 // @TODO: Enable when FSP supports attr sync and core state in HDAT
+      // Story 41245
 
-    //  dump physical path to targets
-    EntityPath l_path;
-    l_path  =   l_@targetN_target->getAttr<ATTR_PHYS_PATH>();
-    l_path.dump();
+    uint64_t l_masterCoreID = task_getcpuid() & ~7;
 
-    // cast OUR type of target to a FAPI type of target.
-    const fapi::Target l_fapi_@targetN_target(
-                    TARGET_TYPE_MEMBUF_CHIP,
-                    reinterpret_cast<void *>
-                        (const_cast<TARGETING::Target*>(l_@targetN_target)) );
+    TargetHandleList l_cores;
+    getAllChiplets(l_cores, TYPE_CORE);
 
-    //  call the HWP with each fapi::Target
-    FAPI_INVOKE_HWP( l_errl, host_activate_slave_cores, _args_...);
-    if ( l_errl )
+    for(TargetHandleList::const_iterator l_core = l_cores.begin();
+        l_core != l_cores.end();
+        ++l_core)
     {
-        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                  "ERROR : .........." );
-        errlCommit( l_errl, HWPF_COMP_ID );
-    }
-    else
-    {
-        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                   "SUCCESS : .........." );
+        ConstTargetHandle_t l_processor = getParentChip(*l_core);
+
+        CHIP_UNIT_ATTR l_coreId =
+                (*l_core)->getAttr<TARGETING::ATTR_CHIP_UNIT>();
+        FABRIC_NODE_ID_ATTR l_logicalNodeId =
+                l_processor->getAttr<TARGETING::ATTR_FABRIC_NODE_ID>();
+        FABRIC_CHIP_ID_ATTR l_chipId =
+                l_processor->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
+
+        uint64_t pir = l_coreId << 3;
+        pir |= l_chipId << 7;
+        pir |= l_logicalNodeId << 10;
+
+        if (pir != l_masterCoreID)
+        {
+            TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                       "call_host_activate_slave_cores: Waking %x", pir);
+
+            int rc = cpu_start_core(pir);
+
+            // We purposefully only create one error log here.  The only
+            // failure from the kernel is a bad PIR, which means we have
+            // a pervasive attribute problem of some sort.  Just log the
+            // first failing PIR.
+            if ((0 != rc) && (NULL == l_errl))
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           "call_host_activate_slave_cores: Error from kernel"
+                           " %d on core %x",
+                           rc, pir);
+                /*@
+                 * @errortype
+                 * @reasoncode  ISTEP_BAD_RC
+                 * @severity    ERRORLOG::ERRL_SEV_UNRECOVERABLE
+                 * @moduleid    ISTEP_ACTIVATE_SLAVE_CORES
+                 * @userdata1   PIR of failing core.
+                 * @userdata2   rc of cpu_start_core().
+                 *
+                 * @devdesc Kernel returned error when trying to activate core.
+                 */
+                l_errl =
+                    new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                            ISTEP_ACTIVATE_SLAVE_CORES,
+                                            ISTEP_BAD_RC,
+                                            pir,
+                                            rc );
+            }
+        }
     }
-    // @@@@@    END CUSTOM BLOCK:   @@@@@
+
 #endif
+    // @@@@@    END CUSTOM BLOCK:   @@@@@
 
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_slave_cores exit" );
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 2afee559b..26104ab6c 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -41,6 +41,7 @@
 #include <vmmconst.h>
 #include <targeting/common/attributes.H>
 #include <devicefw/userif.H>
+#include <sys/time.h>
 
 #define INTR_TRACE_NAME INTR_COMP_NAME
 
@@ -217,14 +218,14 @@ void IntrRp::msgHandler()
                     ext_intr_t type = NO_INTERRUPT;
 
                     // xirr was read by interrupt message handler.
-                    // Passed in as data[0]
-                    uint32_t xirr = static_cast<uint32_t>(msg->data[0]);
-                    PIR_t pir = static_cast<PIR_t>(msg->data[1]);
+                    // Passed in as data[1]
+                    uint32_t xirr = static_cast<uint32_t>(msg->data[1]);
+                    // data[0] has the PIR
+                    PIR_t pir = static_cast<PIR_t>(msg->data[0]);
 
-                    // data[1] has the PIR
+                    uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(pir);
                     uint32_t * xirrAddress =
-                        reinterpret_cast<uint32_t*>(iv_baseAddr + XIRR_OFFSET +
-                                                    cpuOffsetAddr(pir));
+                        reinterpret_cast<uint32_t*>(baseAddr + XIRR_OFFSET);
 
                     // type = XISR = XIRR[8:31]
                     // priority = XIRR[0:7]
@@ -259,6 +260,15 @@ void IntrRp::msgHandler()
                         }
                         msg_free(rmsg);
                     }
+                    else if (type == INTERPROC)
+                    {
+                        // Ignore "spurious" IPIs (handled below).
+
+                        // Note that we could get an INTERPROC interrupt
+                        // and handle it through the above registration list
+                        // as well.  This is to catch the case where no one
+                        // has registered for an IPI.
+                    }
                     else  // no queue registered for this interrupt type
                     {
                         // Throw it away for now.
@@ -269,6 +279,45 @@ void IntrRp::msgHandler()
                                   (uint32_t)type);
                     }
 
+                    // Handle IPIs special since they're used for waking up
+                    // cores and have special clearing requirements.
+                    if (type == INTERPROC)
+                    {
+                        // Clear IPI request.
+                        volatile uint8_t * mfrr =
+                            reinterpret_cast<uint8_t*>(baseAddr + MFRR_OFFSET);
+                        (*mfrr) = 0xff;
+                        eieio();  // Force mfrr clear before xirr EIO.
+
+                        // Deal with pending IPIs.
+                        PIR_t core_pir = pir; core_pir.threadId = 0;
+                        if (iv_ipisPending.count(core_pir))
+                        {
+                            TRACFCOMP(g_trac_intr,INFO_MRK
+                                      "IPI wakeup received for %d", pir.word);
+
+                            IPI_Info_t& ipiInfo = iv_ipisPending[core_pir];
+
+                            ipiInfo.first &= ~(1 << pir.threadId);
+
+                            if (0 == ipiInfo.first)
+                            {
+                                msg_t* ipiMsg = ipiInfo.second;
+                                iv_ipisPending.erase(core_pir);
+
+                                ipiMsg->data[1] = 0;
+                                msg_respond(iv_msgQ, ipiMsg);
+                            }
+                            else
+                            {
+                                TRACDCOMP(g_trac_intr,INFO_MRK
+                                          "IPI still pending for %x",
+                                          ipiInfo.first);
+                            }
+
+                        }
+                    }
+
                     // Writing the XIRR with the same value read earlier
                     // tells the interrupt presenter hardware to signal an EOI.
                     *xirrAddress = xirr;
@@ -330,10 +379,9 @@ void IntrRp::msgHandler()
 
             //  Called when a new cpu becomes active other than the master
             //  Expect a call for each new core
-            case MSG_INTR_ADD_CPU_USR:
             case MSG_INTR_ADD_CPU:
                 {
-                    PIR_t pir = msg->data[0];
+                    PIR_t pir = msg->data[1];
                     pir.threadId = 0;
                     iv_cpuList.push_back(pir);
 
@@ -342,17 +390,15 @@ void IntrRp::msgHandler()
                               pir.nodeId, pir.chipId, pir.coreId,
                               pir.threadId);
 
-
                     size_t threads = cpu_thread_count();
+                    iv_ipisPending[pir] = IPI_Info_t((1 << threads)-1, msg);
 
                     for(size_t thread = 0; thread < threads; ++thread)
                     {
                         pir.threadId = thread;
                         initInterruptPresenter(pir);
+                        sendIPI(pir);
                     }
-
-                    msg->data[1] = 0;
-                    msg_respond(iv_msgQ, msg);
                 }
                 break;
 
@@ -383,7 +429,7 @@ errlHndl_t IntrRp::setBAR(TARGETING::Target * i_target,
     uint64_t barValue = static_cast<uint64_t>(ICPBAR_VAL) +
             (8 * i_pir.nodeId) + i_pir.chipId;
     barValue <<= 34;
-    barValue |= 1ULL << (63 - ICPBAR_EN);  
+    barValue |= 1ULL << (63 - ICPBAR_EN);
 
     TRACFCOMP(g_trac_intr,"INTR: Target %p. ICPBAR value: 0x%016lx",
               i_target,barValue);
@@ -458,11 +504,13 @@ void IntrRp::initInterruptPresenter(const PIR_t i_pir) const
               cpuOffsetAddr(i_pir));
     if(i_pir.word == iv_masterCpu.word)
     {
-        *cppr = 0xff;
+        *cppr = 0xff;      // Allow all interrupts on master.
     }
     else
     {
-        *cppr = 0;      // no interrupts allowed except on master
+        *cppr = 0x01;      // Allow only priority 0 interrupts on non-masters.
+                           // We use priority 0 to deliver IPIs for waking up
+                           // a core.
     }
 
     // Links are intended to be set up in rings.  If an interrupt ends up
@@ -526,6 +574,17 @@ void IntrRp::deconfigureInterruptPresenter(const PIR_t i_pir) const
 }
 
 
+void IntrRp::sendIPI(const PIR_t i_pir) const
+{
+    uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(i_pir);
+    volatile uint8_t * mfrr =
+        reinterpret_cast<uint8_t*>(baseAddr + MFRR_OFFSET);
+
+    eieio(); sync();
+    MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE);
+    (*mfrr) = 0x00;
+}
+
 
 errlHndl_t IntrRp::checkAddress(uint64_t i_addr)
 {
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 8e2122a52..a377d4cdc 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -85,6 +85,7 @@ namespace INTR
                 XIRR_RO_OFFSET = 0,             //!< offset to XIRR (poll)
                 CPPR_OFFSET = 4,                //!< offset to CPPR (1 byte)
                 XIRR_OFFSET = 4,                //!< offset to XIRR (4 bytes)
+                MFRR_OFFSET = 12,               //!< offset to MFRR (12 bytes)
                 LINKA_OFFSET = 16,              //!< offset to LINKA register
                 LINKB_OFFSET = 20,              //!< offset to LINKB register
                 LINKC_OFFSET = 24,              //!< offset to LINKC register
@@ -149,11 +150,16 @@ namespace INTR
                 };
                 PIR_t(uint32_t i_word) : word(i_word) {}
 
-                PIR_t operator = (uint32_t i_word)
+                PIR_t operator= (uint32_t i_word)
                 {
                     word = i_word;
                     return word;
                 }
+
+                bool operator< (const PIR_t& r) const
+                {
+                    return word < r.word;
+                }
             };
 
             struct intr_response_t
@@ -173,6 +179,7 @@ namespace INTR
                  */
                 intr_response_t(msg_q_t i_msgQ, uint32_t i_msgType) :
                     msgQ(i_msgQ), msgType(i_msgType) {}
+
             };
 
 
@@ -186,6 +193,10 @@ namespace INTR
             PIR_t       iv_masterCpu;   //!< Master cpu PIR
             CpuList_t   iv_cpuList;     //!< Other CPU chips
 
+            typedef std::pair<uint8_t, msg_t*> IPI_Info_t;
+            typedef std::map<PIR_t, IPI_Info_t> IPI_Pending_t;
+            IPI_Pending_t iv_ipisPending;  //!< Pending IPIs.
+
 
         private:        //functions
 
@@ -214,7 +225,7 @@ namespace INTR
              * i_msgQ with i_intr_type in message data word 0 and then waits
              * for a response.
              */
-            errlHndl_t registerInterrupt(msg_q_t i_msgQ, 
+            errlHndl_t registerInterrupt(msg_q_t i_msgQ,
                                          uint32_t i_msg_type,
                                          ext_intr_t i_intr_type);
 
@@ -241,6 +252,12 @@ namespace INTR
             void deconfigureInterruptPresenter(const PIR_t i_pir) const;
 
             /**
+             * Issue IPI to a thread.
+             * @param[in] i_pir - The PIR value of the core to send IPI to.
+             */
+            void sendIPI(const PIR_t i_pir) const;
+
+            /**
              * Set the IPCBAR scom register
              * @param[in] i_target, the Target.
              * @param[in] i_pir, The pir for this processor
@@ -261,7 +278,7 @@ namespace INTR
                 ALWAYS_INLINE
                 uint64_t cpuOffsetAddr(const PIR_t i_pir) const
                 {
-                    // TODO when P7 support is removed then change this 
+                    // TODO when P7 support is removed then change this
                     // to use InterruptMsgHdlr::mmio_offset()
                     uint64_t offset = (i_pir.nodeId  * 8) + i_pir.chipId;
                     offset <<= 20;
diff --git a/src/usr/intr/test/intrtest.H b/src/usr/intr/test/intrtest.H
index 7cdfc0d0b..93c2d235a 100644
--- a/src/usr/intr/test/intrtest.H
+++ b/src/usr/intr/test/intrtest.H
@@ -151,7 +151,6 @@ class IntrTest: public CxxTest::TestSuite
             volatile uint8_t * mfrr =
                 reinterpret_cast<uint8_t *>(iv_masterAddr+12);
             *(mfrr) = 0x55;
-            *(mfrr) = 0xff;
 
             msg_t* msg = msg_wait(msgQ); // wait for interrupt msg
             TRACFCOMP(g_trac_intr,"Interrupt handled! Type=%lx",msg->data[0]);
diff --git a/src/usr/testcore/lib/time.H b/src/usr/testcore/lib/time.H
index 895d2884e..0eb090ae4 100644
--- a/src/usr/testcore/lib/time.H
+++ b/src/usr/testcore/lib/time.H
@@ -1,31 +1,32 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/testcore/lib/time.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2012
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
-
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/testcore/lib/time.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 #ifndef __TESTCORE_LIB_TIME_H
 #define __TESTCORE_LIB_TIME_H
 
 #include <time.h>
 #include <sys/time.h>
+#include <sys/task.h>
 #include <errno.h>
 
 /** @file time.H
@@ -40,6 +41,12 @@ class TimeTest : public CxxTest::TestSuite
          */
         void testClockGetTime()
         {
+            task_affinity_pin();  // TODO: Bug in PHYP start-cores code that
+                                  //       does not sync timebase register, so
+                                  //       we need to pin ourselves to a core
+                                  //       while doing timebase testing.
+                                  // RTC: 42921
+
             timespec_t first, second;
 
             if (-EINVAL != clock_gettime(CLOCK_REALTIME, &first))
@@ -64,6 +71,8 @@ class TimeTest : public CxxTest::TestSuite
             {
                 TS_FAIL("Monotonic clock is not increasing.");
             }
+
+            task_affinity_unpin(); // TODO.
         }
 };