diff options
-rw-r--r-- | src/include/kernel/intmsghandler.H | 18 | ||||
-rw-r--r-- | src/include/kernel/misc.H | 36 | ||||
-rw-r--r-- | src/include/sys/misc.h | 24 | ||||
-rw-r--r-- | src/kernel/intmsghandler.C | 17 | ||||
-rw-r--r-- | src/kernel/misc.C | 95 | ||||
-rw-r--r-- | src/kernel/syscall.C | 15 | ||||
-rw-r--r-- | src/lib/syscall_misc.C | 18 |
7 files changed, 208 insertions, 15 deletions
diff --git a/src/include/kernel/intmsghandler.H b/src/include/kernel/intmsghandler.H index 73fbd9313..88e1c2c1e 100644 --- a/src/include/kernel/intmsghandler.H +++ b/src/include/kernel/intmsghandler.H @@ -61,10 +61,11 @@ class InterruptMsgHdlr : public MessageHandler // Logical Shift Left fields for mmio Base address from PIR. // (IP addr bit pos - PIR bit pos) P8_IP_THREADID_LSL = (12-0), - P8_IP_COREID_LSL = (15-3), + P8_IP_COREID_LSL = (15-3), P8_IP_CHIPID_LSL = (20-7), P8_IP_NODEID_LSL = (22-10), XIRR_ADDR_OFFSET = 4, + MFRR_ADDR_OFFSET = 12, }; // Notes: @@ -77,7 +78,7 @@ class InterruptMsgHdlr : public MessageHandler InterruptMsgHdlr(MessageQueue * i_msgQ) : MessageHandler(NULL,i_msgQ) {} - /** + /** * Destructor. */ virtual ~InterruptMsgHdlr() {}; @@ -104,10 +105,10 @@ class InterruptMsgHdlr : public MessageHandler // The PIR chip id field has 1 extra bit (8 chips), so we need // to shift the node and chip separately - offset |= + offset |= (i_pir & P8_PIR_NODEID_MSK) << P8_IP_NODEID_LSL; - offset |= + offset |= (i_pir & P8_PIR_CHIPID_MSK) << P8_IP_CHIPID_LSL; // The core and thread id field are adjacent in both the PIR and @@ -118,7 +119,7 @@ class InterruptMsgHdlr : public MessageHandler return offset; } - + /** * Create the InterruptMsgHdlr to handle external interrupts * @param[in] i_msgQ The message queue @@ -138,6 +139,13 @@ class InterruptMsgHdlr : public MessageHandler */ static void addCpuCore(uint64_t i_pir); + /** + * Issue an IPI to the core. + * + * @param[in] i_pir - The PIR of the CPU to send IPI at. + */ + static void sendIPI(uint64_t i_pir); + private: static InterruptMsgHdlr * cv_instance; diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H index 11677d250..f0fe0ffd0 100644 --- a/src/include/kernel/misc.H +++ b/src/include/kernel/misc.H @@ -90,6 +90,37 @@ namespace KernelMisc uint64_t iv_timebase; }; + /** @class WinkleAll + * + * @brief DeferredWork class for implementing cpu_all_winkle. + */ + class WinkleAll : public ::DeferredWork + { + public: + /** ctor + * + * @param i_caller - Task calling cpu_all_winkle to be restored + * after winkle is complete. + * + */ + WinkleAll(task_t* i_caller) : + iv_caller(i_caller), iv_timebase(0), iv_firstThread(0) {}; + + void masterPreWork(); + void activeMainWork(); + void masterPostWork(); + void nonactiveMainWork(); + + private: + /** Caller of cpu_all_winkle() */ + task_t* iv_caller; + /** Current time-base value for restore after winkle. */ + uint64_t iv_timebase; + /** Sync value for first thread to wake up. */ + uint64_t iv_firstThread; + + }; + /** @fn expand_full_cache * * @brief Expands the image footprint from a half-cache to full-cache @@ -123,14 +154,13 @@ namespace KernelMisc * * NOTE: This function is a wrapper function for writeScratchReg that * takes care of modifying the scratch register address value depending - * on the getCPUType. The writeScratchReg takes the scratch address + * on the getCPUType. The writeScratchReg takes the scratch address * passed in and puts that date in that register using assembly code * * @param[in] uint64_t - scratch_addr - * @param[in] uint64_t - Data + * @param[in] uint64_t - Data */ void updateScratchReg(MMIO_Scratch_Register scratch_addr, uint64_t data); - }; #endif diff --git a/src/include/sys/misc.h b/src/include/sys/misc.h index a4ae8d006..d680548b8 100644 --- a/src/include/sys/misc.h +++ b/src/include/sys/misc.h @@ -58,6 +58,17 @@ enum ShutdownStatus SHUTDOWN_STATUS_INITSVC_FAILED = 0x01230004, }; +/** + * @enum WinkleScopes + * + * Scope of the winkle operation. + */ +enum WinkleScope +{ + WINKLE_SCOPE_MASTER = 0x0, + WINKLE_SCOPE_ALL = 0x1, +}; + #ifdef __cplusplus extern "C" { @@ -161,6 +172,19 @@ uint64_t cpu_spr_value(CpuSprNames spr); */ int cpu_master_winkle(); +/** @fn cpu_all_winkle + * @brief Winkle all the threads. + * + * This is used in multi-node systems to quiesce all the cores in a drawer + * prior to the fabric being stitched together. + * + * @retval 0 - Success + * + * @note This function will migrate the task to the master thread and in the + * process will unset any task affinity. See task_affinity_unpin(). + */ +int cpu_all_winkle(); + /** @fn cpu_crit_assert * @brief Forces a Terminate Immediate after a crit-assert is issued * @param[in] i_failAddr - value in the linkRegister of the address diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C index a2f2fb096..538d322ae 100644 --- a/src/kernel/intmsghandler.C +++ b/src/kernel/intmsghandler.C @@ -5,7 +5,7 @@ /* */ /* IBM CONFIDENTIAL */ /* */ -/* COPYRIGHT International Business Machines Corp. 2011,2012 */ +/* COPYRIGHT International Business Machines Corp. 2011,2013 */ /* */ /* p1 */ /* */ @@ -76,7 +76,7 @@ void InterruptMsgHdlr::handleInterrupt() if( cv_ipc_base_address != 0 ) { - uint64_t xirrAddress = cv_ipc_base_address; + uint64_t xirrAddress = cv_ipc_base_address; xirrAddress += mmio_offset(pir); // Add offset for this cpu xirrAddress += XIRR_ADDR_OFFSET; // Add offset for XIRR register @@ -144,6 +144,19 @@ void InterruptMsgHdlr::addCpuCore(uint64_t i_pir) } } +void InterruptMsgHdlr::sendIPI(uint64_t i_pir) +{ + uint64_t mfrrAddress = cv_ipc_base_address; + mfrrAddress += mmio_offset(i_pir); + mfrrAddress += MFRR_ADDR_OFFSET; + + register uint8_t data = 0; + + eieio(); sync(); + MAGIC_INSTRUCTION(MAGIC_SIMICS_CORESTATESAVE); + asm volatile("stbcix %0,0,%1" :: "r" (data) , "r" (mfrrAddress)); +} + MessageHandler::HandleResult InterruptMsgHdlr::handleResponse ( msg_sys_types_t i_type, diff --git a/src/kernel/misc.C b/src/kernel/misc.C index f73bf2947..9fd511c33 100644 --- a/src/kernel/misc.C +++ b/src/kernel/misc.C @@ -34,6 +34,7 @@ #include <kernel/pagemgr.H> #include <kernel/vmmmgr.H> // INITIAL_MEM_SIZE #include <kernel/memstate.H> +#include <kernel/intmsghandler.H> extern "C" void kernel_shutdown(size_t, uint64_t, uint64_t, uint64_t) NO_RETURN; @@ -66,7 +67,7 @@ namespace KernelMisc status); } - // Call to set the Core Scratch Reg 0 with the status + // Call to set the Core Scratch Reg 0 with the status updateScratchReg(MMIO_SCRATCH_PROGRESS_CODE, status); } @@ -223,6 +224,98 @@ namespace KernelMisc kassert(false); } + void WinkleAll::masterPreWork() + { + printk("Winkle all - "); + + // Save away the current timebase. All threads are in this object + // now so they're not going to be using the time for anything else. + iv_timebase = getTB(); + } + + void WinkleAll::activeMainWork() + { + cpu_t* cpu = CpuManager::getCurrentCPU(); + + // Return current task to run-queue so it isn't lost. + cpu->scheduler->returnRunnable(); + TaskManager::setCurrentTask(cpu->idle_task); + + // Clear LPCR values that wakes up from winkle. LPCR[49, 50, 51] + // Otherwise, there may be an interrupt pending or something that + // prevents us from fully entering winkle. + setLPCR(getLPCR() & (~0x0000000000007000)); + + // Deactivate CPU from kernel. + cpu->winkled = true; + CpuManager::deactivateCPU(cpu); + + // Create kernel save area and store ptr in bottom of kernel stack. + task_t* saveArea = new task_t; + memset(saveArea, '\0', sizeof(task_t)); + saveArea->context.msr_mask = 0xD030; // EE, ME, PR, IR, DR. + *(reinterpret_cast<task_t**>(cpu->kernel_stack_bottom)) = saveArea; + + // Execute winkle. + kernel_execute_winkle(saveArea); + + // Re-activate CPU in kernel and re-init VMM SPRs. + delete saveArea; + cpu->winkled = false; + CpuManager::activateCPU(cpu); + VmmManager::init_slb(); + + // Wake up all the other threads. + if(__sync_bool_compare_and_swap(&iv_firstThread, 0, 1)) + { + for(uint64_t i = 0; i < KERNEL_MAX_SUPPORTED_CPUS_PER_NODE * + KERNEL_MAX_SUPPORTED_NODES; i++) + { + cpu_t* slave = CpuManager::getCpu(i); + + if ((NULL != slave) && (slave != cpu)) + { + if (slave->winkled) + { + InterruptMsgHdlr::sendIPI(i); + } + } + } + }; + + // Sync timebase. + if (getTB() < iv_timebase) + { + setTB(iv_timebase); + } + + // Select a new task if not the master CPU. Master CPU will resume + // the code that called cpu_master_winkle(). + if (!cpu->master) + { + cpu->scheduler->setNextRunnable(); + } + + } + + void WinkleAll::masterPostWork() + { + printk("Awake!\n"); + + // Restore caller of cpu_all_winkle(). + iv_caller->state = TASK_STATE_RUNNING; + TaskManager::setCurrentTask(iv_caller); + } + + void WinkleAll::nonactiveMainWork() + { + // Race condition that should not occur... + // + // Attempted to winkle the threads and another thread came online in + // the process. + kassert(false); + } + int expand_full_cache() { static bool executed = false; diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C index 71bd1a995..985d9c0ab 100644 --- a/src/kernel/syscall.C +++ b/src/kernel/syscall.C @@ -700,12 +700,13 @@ namespace Systemcalls } }; - /** Winkle all the threads so that the runtime SLW image can be loaded. */ + /** Winkle all the threads. */ void CpuWinkle(task_t *t) { cpu_t* cpu = CpuManager::getCurrentCPU(); - if ((CpuManager::getCpuCount() > CpuManager::getThreadCount()) || + if ((WINKLE_SCOPE_MASTER == TASK_GETARG0(t) && + (CpuManager::getCpuCount() > CpuManager::getThreadCount())) || (!cpu->master)) { TASK_SETRTN(t, -EDEADLK); @@ -713,7 +714,15 @@ namespace Systemcalls else { TASK_SETRTN(t, 0); - KernelMisc::WinkleCore* deferred = new KernelMisc::WinkleCore(t); + DeferredWork* deferred = NULL; + if (WINKLE_SCOPE_MASTER == TASK_GETARG0(t)) + { + deferred = new KernelMisc::WinkleCore(t); + } + else + { + deferred = new KernelMisc::WinkleAll(t); + } t->state = TASK_STATE_BLOCK_USRSPACE; t->state_info = deferred; DeferredQueue::insert(deferred); diff --git a/src/lib/syscall_misc.C b/src/lib/syscall_misc.C index 7f490e2ac..f2b50bb01 100644 --- a/src/lib/syscall_misc.C +++ b/src/lib/syscall_misc.C @@ -85,7 +85,23 @@ int cpu_master_winkle() task_affinity_pin(); task_affinity_migrate_to_master(); - int rc = reinterpret_cast<int64_t>(_syscall0(MISC_CPUWINKLE)); + int rc = reinterpret_cast<int64_t>( + _syscall1(MISC_CPUWINKLE, + reinterpret_cast<void*>(WINKLE_SCOPE_MASTER))); + + task_affinity_unpin(); + + return rc; +} + +int cpu_all_winkle() +{ + task_affinity_pin(); + task_affinity_migrate_to_master(); + + int rc = reinterpret_cast<int64_t>( + _syscall1(MISC_CPUWINKLE, + reinterpret_cast<void*>(WINKLE_SCOPE_ALL))); task_affinity_unpin(); |