HOSTBOOT: Support fused cores

Change-Id: I2ad133be733ee9e41590b3b8bd60bd6abe69d1a9
RTC: 126786
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/22054
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

16 files changed, 290 insertions, 37 deletions

diff --git a/src/build/debug/simics-debug-framework.py b/src/build/debug/simics-debug-framework.py
index de639f4fb..24dee4eeb 100755
--- a/src/build/debug/simics-debug-framework.py
+++ b/src/build/debug/simics-debug-framework.py
@@ -6,7 +6,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2011,2014
+# Contributors Listed Below - COPYRIGHT 2011,2015
 # [+] Google Inc.
 # [+] International Business Machines Corp.
 #
@@ -513,6 +513,61 @@ def magic_instruction_callback(user_arg, cpu, arg):
         cmd = 'shell "fcp --force -o0 -R %s:PAYLOAD simicsPayload.ecc; ecc --remove --p8 simicsPayload.ecc simicsPayload"; load-file simicsPayload 0x%x' % (flash_file, load_addr)
         SIM_run_alone( run_command, cmd )
 
+    if arg == 7015:
+        print "Wake up master Thread CORE0 - 7015 start \n";
+
+        # If more than 1 thread, we only want to do
+        # this on 4th hit -- won't worry about it for now.
+        setIar0 = "system_cmp0.cpu0_0_00_0->iar=0x100"
+        SIM_run_alone(run_command, setIar0 )
+
+        # need to disable and then re-enable thread0 to
+        # make it appear to wake back up
+        disable0 = "system_cmp0.cpu0_0_00_0.disable"
+        SIM_run_alone(run_command, disable0 )
+        enable0 =  "system_cmp0.cpu0_0_00_0.enable"
+        SIM_run_alone(run_command, enable0 )
+        print "Master Thread should be awake now \n";
+
+    if arg == 7016:
+        print "Wake up rest of CORE0 - 7017 start \n";
+
+        # Set other threads on CORE0 to proper address
+        setIar1 = "system_cmp0.cpu0_0_00_1->iar=0x100"
+        setIar2 = "system_cmp0.cpu0_0_00_2->iar=0x100"
+        setIar3 = "system_cmp0.cpu0_0_00_3->iar=0x100"
+
+        setIarAll = "%s; %s; %s"%(setIar1,setIar2,setIar3)
+        SIM_run_alone(run_command, setIarAll )
+
+        # Enable rest of threads on CORE0
+        enable1 =  "system_cmp0.cpu0_0_00_1.enable"
+        enable2 =  "system_cmp0.cpu0_0_00_2.enable"
+        enable3 =  "system_cmp0.cpu0_0_00_3.enable"
+        enableCore0 = "%s; %s; %s"%(enable1,enable2,enable3)
+        SIM_run_alone(run_command, enableCore0 )
+
+    if arg == 7017:
+        print "Wake up FUSED threads - 7017 start \n";
+
+        # Set CORE1 threads to proper address
+        setIar4  = "system_cmp0.cpu0_0_01_0->iar=0x100"
+        setIar5  = "system_cmp0.cpu0_0_01_1->iar=0x100"
+        setIar6  = "system_cmp0.cpu0_0_01_2->iar=0x100"
+        setIar7  = "system_cmp0.cpu0_0_01_3->iar=0x100"
+        setHrmor = "system_cmp0.cpu0_0_01_0->hrmor=0x08000000"
+
+        setIarAll = "%s; %s; %s; %s; %s"%(setIar4,setIar5,setIar6,setIar7,setHrmor)
+        SIM_run_alone(run_command, setIarAll )
+
+        # Enable all threads on CORE1
+        enable4 =  "system_cmp0.cpu0_0_01_0.enable"
+        enable5 =  "system_cmp0.cpu0_0_01_1.enable"
+        enable6 =  "system_cmp0.cpu0_0_01_2.enable"
+        enable7 =  "system_cmp0.cpu0_0_01_3.enable"
+        enableCore1 = "%s; %s; %s; %s"%(enable4,enable5,enable6,enable7)
+        SIM_run_alone(run_command, enableCore1 )
+
     if arg == 7055:   # MAGIC_CONTINUOUS_TRACE
         hb_tracBinaryBuffer = cpu.r4
         hb_tracBinaryBufferSz = cpu.r5
@@ -605,4 +660,3 @@ SIM_hap_add_callback_range( "Core_Magic_Instruction", magic_instruction_callback
 
 # Run the registration automatically whenever this script is loaded.
 register_hb_debug_framework_tools()
-
diff --git a/src/include/arch/ppc.H b/src/include/arch/ppc.H
index 8402f743c..455728500 100644
--- a/src/include/arch/ppc.H
+++ b/src/include/arch/ppc.H
@@ -339,6 +339,29 @@ inline void nap()
 {
     // @todo-RTC:130186 Add new stop command support
     //asm volatile("nap");
+
+    // Nap should be equivalent to 'stop 1' (no state loss)
+    // 855 reg for HYPV,  823 reg otherwise
+    // using EC(bit43) as 1 here  (system reset or LPCR event)
+    // PHYP uses bit 42 as 1 also
+//  register uint64_t psscr = 0x0000000000310001;
+//  asm volatile("mtspr 823, %0; isync" :: "r" (psscr));
+//  asm volatile(".long 0x4C0002E4"); // When GCC supports 'stop', use it
+}
+
+ALWAYS_INLINE
+inline void setPSSCR(uint64_t _psscr)
+{
+    register uint64_t psscr = _psscr;
+    asm volatile("mtspr 855, %0; isync" :: "r" (psscr));
+}
+
+ALWAYS_INLINE
+inline uint64_t getPSSCR()
+{
+    register uint64_t psscr = 0;
+    asm volatile("mfspr %0, 855" : "=r" (psscr));
+    return psscr;
 }
 
 ALWAYS_INLINE
@@ -426,6 +449,10 @@ enum
                                      // some state from the core doing the
                                      // wakeup to apply into the woken one.
 
+    MAGIC_SIMICS_FUSEDCOREWAKE = 11, // Indicate to the PHYP model of simics
+                                     // that we are waking up and expecting
+                                     // CORES to become fused.
+
     MAGIC_SHUTDOWN            =  7006, // KernelMisc::shutdown() called.
     MAGIC_BREAK               =  7007, // hard-code a breakpoint
     MAGIC_RANDOM              =  7008, // generate random number
@@ -433,6 +460,13 @@ enum
     MAGIC_FAKEPAYLOAD_ENTER   =  7010, // Entered the fake payload.
     MAGIC_SIMICS_CHECK        =  7011, // Check if system is running on simics
     MAGIC_LOAD_PAYLOAD        =  7012, // load payload from flash
+
+                                       // These are used for getting threads
+                                       // going after doing 'stop/winkle'
+    MAGIC_WAKE_MASTER_THREAD  =  7015, // Wake master thread
+    MAGIC_WAKE_OTHER_THREADS  =  7016, // Wake other threads on initial CORE
+    MAGIC_WAKE_FUSED_THREADS  =  7017, // Wake up fused core threads
+
     MAGIC_CONTINUOUS_TRACE    =  7055, // extract mixed trace buffer
 };
 
diff --git a/src/include/kernel/cpumgr.H b/src/include/kernel/cpumgr.H
index 68f897278..30009da73 100644
--- a/src/include/kernel/cpumgr.H
+++ b/src/include/kernel/cpumgr.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2010,2014              */
+/* Contributors Listed Below - COPYRIGHT 2010,2015                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -184,14 +186,15 @@ class CpuManager
          */
         static const uint64_t WAKEUP_RPR_VALUE =    0x0001032021223F;
 
-    protected:
-        CpuManager();
-        ~CpuManager() {}
-
         /** @fn startCPU
          *  Starts the requested CPU. Default of -1 implies current CPU.
          */
         void startCPU(ssize_t i = -1);
+
+    protected:
+        CpuManager();
+        ~CpuManager() {}
+
         void startSlaveCPU(cpu_t*);
 
     private:
diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H
index 2d24c9d85..802df9090 100644
--- a/src/include/kernel/misc.H
+++ b/src/include/kernel/misc.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2015                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -93,9 +95,11 @@ namespace KernelMisc
              *
              *  @param i_caller - Task calling cpu_master_winkle to be
              *                    restored after winkle is complete.
+             *  @param i_fusedCores - true if using 8 threaded cores 
              */
-            WinkleCore(task_t* i_caller) :
-                iv_caller(i_caller), iv_timebase(0) {};
+            WinkleCore(task_t* i_caller, bool i_fusedCores) :
+                iv_caller(i_caller), iv_timebase(0),
+                iv_fusedCores(i_fusedCores) {};
 
             void masterPreWork();
             void activeMainWork();
@@ -107,6 +111,9 @@ namespace KernelMisc
             task_t* iv_caller;
                 /** Current time-base value for restore after winkle. */
             uint64_t iv_timebase;
+                /** Fused core has 8 threads per core (normal is 4).   */
+                /** So when winkling core, we need a 2nd core started. */
+            bool  iv_fusedCores;
     };
 
     /** @class WinkleAll
diff --git a/src/include/sys/misc.h b/src/include/sys/misc.h
index 625b1a78b..74f5ac46a 100644
--- a/src/include/sys/misc.h
+++ b/src/include/sys/misc.h
@@ -191,13 +191,15 @@ uint64_t cpu_spr_value(CpuSprNames spr);
  *  Will execute the winkle instruction on all running threads and return when
  *  an IPI is receieved on the master thread of the core.
  *
+ *  @param[in] i_fusedCores - Fused cores if true,  Regular cores if false
+ *
  *  @retval 0 - Success
  *  @retval -EDEADLK - Cores other than the master are already running.
  *
  *  @note This function will migrate the task to the master thread and in the
  *        process will unset any task affinity.  See task_affinity_unpin().
  */
-int cpu_master_winkle();
+int cpu_master_winkle(bool  i_fusedCores);
 
 /** @fn cpu_all_winkle
  *  @brief Winkle all the threads.
diff --git a/src/include/usr/isteps/istep16list.H b/src/include/usr/isteps/istep16list.H
index 3effb1adf..4f5ab931c 100644
--- a/src/include/usr/isteps/istep16list.H
+++ b/src/include/usr/isteps/istep16list.H
@@ -111,13 +111,13 @@ namespace   INITSERVICE
                 NULL,
                 { NONE, EXT_IMAGE, IPL_NOOP, false }
         },
-// @todo RTC:137389 - removed for P9 bringup
-#if (0)
         {
                 ISTEPNAME(16,01,"host_activate_master"),
                 ISTEP_16::call_host_activate_master,
                 { START_FN, EXT_IMAGE, NORMAL_IPL_OP | MPIPL_OP, true }
         },
+// @todo RTC:137389 - removed for P9 bringup
+#if 0
         {
                 ISTEPNAME(16,02,"host_activate_slave_cores"),
                 ISTEP_16::call_host_activate_slave_cores,
diff --git a/src/include/usr/targeting/common/util.H b/src/include/usr/targeting/common/util.H
index 29028cc80..1ec88be61 100644
--- a/src/include/usr/targeting/common/util.H
+++ b/src/include/usr/targeting/common/util.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2015                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -161,6 +163,13 @@ uint64_t get_bottom_mem_addr(void);
 bool orderByNodeAndPosition(  Target* i_firstProc,
                               Target* i_secondProc);
 
+/**
+ * @brief   Checks if we want to be in FUSED mode or not.
+ * @description  Fused mode is when 2 cores merge to produce
+ *               a core with 8 threads versus 4 threaded cores.
+ * @return  Non-zero if FUSED mode, 0 if non-FUSED mode
+ */
+uint8_t  is_fused_mode( );
 
 }
 
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index a4b438ceb..080c0adf3 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -40,6 +40,7 @@
 #include <kernel/hbdescriptor.H>
 #include <kernel/ipc.H>
 #include <kernel/timemgr.H>
+#include <util/singleton.H>
 
 
 extern "C"
@@ -249,14 +250,35 @@ namespace KernelMisc
         // Save away the current timebase.  All threads are in this object
         // now so they're not going to be using the time for anything else.
         iv_timebase = getTB() + TimeManager::convertSecToTicks(1,0);
+
+        if (true == iv_fusedCores)
+        {
+            uint64_t  l_numThreads = CpuManager::getThreadCount();
+            cpu_t *   l_cput = CpuManager::getCurrentCPU();
+
+            // creates cpu_t structure in advance for new threads
+            // (should be next set of cpuIds past master core)
+            for ( uint64_t  l_threadNum = l_numThreads;
+                  (l_threadNum < (l_numThreads *2));
+                   l_threadNum++ )
+            {
+                Singleton<CpuManager>::instance().startCPU(l_cput->cpu +
+                                                           l_threadNum);
+            }
+
+            // Tell SIMICS we expect more threads (one more core)
+            // to appear after doing the 'stop' instruction.
+            MAGIC_INSTRUCTION(MAGIC_SIMICS_FUSEDCOREWAKE);
+        } // end if fused core mode
     }
 
-    extern "C" void kernel_execute_winkle(task_t* t);
+    extern "C" void kernel_execute_stop(task_t* t);
 
     void WinkleCore::activeMainWork()
     {
         cpu_t* cpu = CpuManager::getCurrentCPU();
-        printk("%d", static_cast<int>(cpu->cpu & 0x7));
+        ssize_t  l_numThreads = CpuManager::getThreadCount();
+        printk("%d", static_cast<int>(cpu->cpu & (l_numThreads-1)));
 
         // Return current task to run-queue so it isn't lost.
         cpu->scheduler->returnRunnable();
@@ -276,10 +298,14 @@ namespace KernelMisc
         saveArea->context.msr_mask = 0xD030; // EE, ME, PR, IR, DR.
         *(reinterpret_cast<task_t**>(cpu->kernel_stack_bottom)) = saveArea;
 
+        // Set register to indicate we want a 'stop 15' to occur (state loss)
+        uint64_t l_psscr_saved = getPSSCR();
+        setPSSCR( 0x00000000003F00FF );
         // Execute winkle.
-        kernel_execute_winkle(saveArea);
+        kernel_execute_stop(saveArea);
 
         // Re-activate CPU in kernel and re-init VMM SPRs.
+        setPSSCR(l_psscr_saved);
         delete saveArea;
         cpu->winkled = false;
         CpuManager::activateCPU(cpu);
@@ -314,6 +340,39 @@ namespace KernelMisc
 
         //Issue sbe master workaround
         InterruptMsgHdlr::issueSbeMboxWA();
+        // NOTE: The cpu_t structures for theads 1:3 were created
+        //       during init (CpuManager::init).
+
+        #ifdef HOSTBOOT_REAL_WINKLE
+        // @todo- RTC 141924 Start the other threads 1:3 in a new manner
+        //  SBE won't start them and we can't use normal instruction start.
+        //  Maybe something like:   sendIPI(..) or  addCpuCore(..)
+        //  Need interrupt code in place for this.
+
+        #else
+        // get other 3 threads going in SIMICs for now
+        MAGIC_INSTRUCTION(MAGIC_WAKE_OTHER_THREADS);
+        #endif
+
+        if (true == iv_fusedCores)
+        {
+            // If using FUSED cores, we need to essentially
+            // treat this as a new core appearing.
+
+            #ifdef HOSTBOOT_REAL_WINKLE
+            ssize_t  l_numThreads = CpuManager::getThreadCount();
+            cpu_t*   l_cpuPtr = CpuManager::getCurrentCPU();
+            cpuid_t  l_startPir = l_cpuPtr->cpu  & (~(l_numThreads - 1));
+
+            // New core should have threads just past current ones
+            CpuManager::startCore(l_startPir + l_numThreads,
+                                  0xF000000000000000);  // all 4 threads
+            #else
+            // get new core going in SIMICS
+            MAGIC_INSTRUCTION(MAGIC_WAKE_FUSED_THREADS);
+            #endif
+        } // end if fused core mode
+
     }
 
     void WinkleCore::nonactiveMainWork()
@@ -356,10 +415,14 @@ namespace KernelMisc
         saveArea->context.msr_mask = 0xD030; // EE, ME, PR, IR, DR.
         *(reinterpret_cast<task_t**>(cpu->kernel_stack_bottom)) = saveArea;
 
+        // Set register to indicate we want a 'stop 15' to ocur (state loss)
+        uint64_t l_psscr_saved = getPSSCR();
+        setPSSCR( 0x00000000003F00FF );
         // Execute winkle.
-        kernel_execute_winkle(saveArea);
+        kernel_execute_stop(saveArea);
 
         // Re-activate CPU in kernel and re-init VMM SPRs.
+        setPSSCR(l_psscr_saved);
         delete saveArea;
         cpu->winkled = false;
         CpuManager::activateCPU(cpu);
diff --git a/src/kernel/start.S b/src/kernel/start.S
index 13351b893..dbb499751 100644
--- a/src/kernel/start.S
+++ b/src/kernel/start.S
@@ -867,15 +867,15 @@ UNIMPL_INTERRUPT_NOADDR(vector_unavail, 0xF20)
 UNIMPL_INTERRUPT_NOADDR(vsx_unavail, 0xF40)
 UNIMPL_INTERRUPT_NOADDR(fac_unavail, 0xF60)
 
-    ;// @fn kernel_execute_winkle
+    ;// @fn kernel_execute_stop
     ;//
     ;// Saves kernel state into a specified task structure and then executes
-    ;// the winkle instruction.
+    ;// the stop instruction.
     ;//
     ;// @param r3 - task_t* to save kernel state into.
     ;//
-.global kernel_execute_winkle
-kernel_execute_winkle:
+.global kernel_execute_stop
+kernel_execute_stop:
     ;// Move save area to SPRG3 for kernel_save_task.
     mtsprg3 r3
 
@@ -883,7 +883,7 @@ kernel_execute_winkle:
     mflr r3
     mtsrr0 r3
 
-    ;// Load winkle instruction address into the "return to" address (SPRG0).
+    ;// Load stop instruction address into the "return to" address (SPRG0).
     lis r3, 1f@h
     ori r3, r3, 1f@l
     mtsprg0 r3
@@ -891,9 +891,12 @@ kernel_execute_winkle:
     ;// Save kernel state.
     b kernel_save_task
 
-    ;// Execute winkle.
+    ;// Execute stop.
 1:
-    rvwinkle
+    ;// Magic instruction(7015) to allow simics to trigger thread wakeup
+    rlwimi  r27,r27,0,6,23
+    ;// When GCC supports 'stop', you can  use it
+    .long 0x4C0002E4
 
 .section .data
     .balign 1024
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index 1b1bd7f57..92907c3df 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -751,7 +751,8 @@ namespace Systemcalls
             DeferredWork* deferred = NULL;
             if (WINKLE_SCOPE_MASTER == TASK_GETARG0(t))
             {
-                deferred = new KernelMisc::WinkleCore(t);
+                bool  l_fusedCores = (bool)TASK_GETARG1(t);
+                deferred = new KernelMisc::WinkleCore(t, l_fusedCores);
             }
             else
             {
diff --git a/src/lib/syscall_misc.C b/src/lib/syscall_misc.C
index 285fb6091..22ead7a9a 100644
--- a/src/lib/syscall_misc.C
+++ b/src/lib/syscall_misc.C
@@ -91,14 +91,16 @@ uint64_t cpu_spr_value(CpuSprNames spr)
         _syscall1(MISC_CPUSPRVALUE, reinterpret_cast<void*>(spr)));
 }
 
-int cpu_master_winkle()
+int cpu_master_winkle(bool  i_fusedCores)
 {
     task_affinity_pin();
     task_affinity_migrate_to_master();
 
     int rc = reinterpret_cast<int64_t>(
-                _syscall1(MISC_CPUWINKLE,
-                          reinterpret_cast<void*>(WINKLE_SCOPE_MASTER)));
+                _syscall2(MISC_CPUWINKLE,
+                          reinterpret_cast<void*>(WINKLE_SCOPE_MASTER),
+                          reinterpret_cast<void*>(i_fusedCores) ));
+
 
     task_affinity_unpin();
 
@@ -111,8 +113,9 @@ int cpu_all_winkle()
     task_affinity_migrate_to_master();
 
     int rc = reinterpret_cast<int64_t>(
-                _syscall1(MISC_CPUWINKLE,
-                          reinterpret_cast<void*>(WINKLE_SCOPE_ALL)));
+                _syscall2(MISC_CPUWINKLE,
+                          reinterpret_cast<void*>(WINKLE_SCOPE_ALL),
+                          reinterpret_cast<void*>(false) ));
 
     task_affinity_unpin();
 
diff --git a/src/usr/isteps/istep16/call_host_activate_master.C b/src/usr/isteps/istep16/call_host_activate_master.C
index a08f942f8..64dd4d71e 100644
--- a/src/usr/isteps/istep16/call_host_activate_master.C
+++ b/src/usr/isteps/istep16/call_host_activate_master.C
@@ -66,7 +66,6 @@ namespace ISTEP_16
 void* call_host_activate_master (void *io_pArgs)
 {
     IStepError  l_stepError;
-#if 0
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_master entry" );
@@ -77,6 +76,7 @@ void* call_host_activate_master (void *io_pArgs)
 
     do  {
 
+#if 0
         // find the master core, i.e. the one we are running on
         TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "call_host_activate_master: Find master core: " );
@@ -205,6 +205,7 @@ void* call_host_activate_master (void *io_pArgs)
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
                       "Disable special wakeup on master core SUCCESS");
         }
+#endif
 
 
         //  put the master into winkle.
@@ -213,8 +214,9 @@ void* call_host_activate_master (void *io_pArgs)
 
         // Flush any lingering console traces first
         CONSOLE::flush();
+        bool l_fusedCores = is_fused_mode();
 
-        int l_rc    =   cpu_master_winkle( );
+        int l_rc    =   cpu_master_winkle(l_fusedCores);
         if ( l_rc )
         {
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
@@ -226,15 +228,15 @@ void* call_host_activate_master (void *io_pArgs)
              * @severity    ERRORLOG::ERRL_SEV_UNRECOVERABLE
              * @moduleid    ISTEP_HOST_ACTIVATE_MASTER
              * @userdata1   return code from cpu_master_winkle
+             * @userdata2   Fused core indicator
              *
-             * @devdesc p8_pore_gen_cpureg returned an error when
-             *          attempting to change a reg value in the PORE image.
+             * @devdesc cpu_master_winkle returned an error 
              */
             l_errl =
             new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                     ISTEP_HOST_ACTIVATE_MASTER,
                                     ISTEP_FAIL_MASTER_WINKLE_RC,
-                                    l_rc  );
+                                    l_rc, l_fusedCores );
             break;
         }
 
@@ -245,6 +247,7 @@ void* call_host_activate_master (void *io_pArgs)
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Returned from Winkle." );
 
+#if (0)
         //Re-enable the mailbox
         l_errl = MBOX::resume();
         if (l_errl)
@@ -315,6 +318,7 @@ void* call_host_activate_master (void *io_pArgs)
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
                       "Enable special wakeup on master core SUCCESS");
         }
+#endif
 
     }   while ( 0 );
 
@@ -330,7 +334,6 @@ void* call_host_activate_master (void *io_pArgs)
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_master exit" );
-#endif
     // end task, returning any errorlogs to IStepDisp
     return l_stepError.getErrorHandle();
 
diff --git a/src/usr/targeting/common/util.C b/src/usr/targeting/common/util.C
index b44b820a8..dd4a7082d 100644
--- a/src/usr/targeting/common/util.C
+++ b/src/usr/targeting/common/util.C
@@ -310,4 +310,38 @@ bool orderByNodeAndPosition(  Target* i_firstProc,
     return nodeId0 < nodeId1;
 }
 
+uint8_t  is_fused_mode( )
+{
+    uint8_t  l_fused;
+    TARGETING::Target * sys = NULL;
+    TARGETING::targetService().getTopLevelTarget( sys );
+    assert(sys != NULL);
+    TARGETING::PAYLOAD_KIND l_payload = sys->getAttr<ATTR_PAYLOAD_KIND>();
+    uint8_t  l_attrValue = sys->getAttr<ATTR_FUSED_CORE_OPTION>();
+
+
+    if (FUSED_CORE_OPTION_USING_DEFAULT_CORES == l_attrValue)
+    {
+        // if payload is PHYP, use FUSED mode
+        // Anything else, use NORMAL mode
+        if (PAYLOAD_KIND_PHYP == l_payload)
+        {
+            l_fused = true;
+        }
+        else
+        {
+            l_fused = false;
+        }
+    }
+    else
+    {
+        l_fused = (l_attrValue == FUSED_CORE_OPTION_USING_NORMAL_CORES) ?
+                   false : true;
+    }
+
+
+    return(l_fused);
+
+} // end is_fused_mode
+
 }
diff --git a/src/usr/targeting/common/xmltohb/attribute_types.xml b/src/usr/targeting/common/xmltohb/attribute_types.xml
index 0aedf3831..748ea9b03 100644
--- a/src/usr/targeting/common/xmltohb/attribute_types.xml
+++ b/src/usr/targeting/common/xmltohb/attribute_types.xml
@@ -16962,4 +16962,36 @@ firmware notes: Platforms should initialize this attribute to AUTO (0)</descript
     <readable/>
 </attribute>
 
+<enumerationType>
+    <id>FUSED_CORE_OPTION</id>
+    <description>Enum for FUSED_CORE_OPTION</description>
+    <enumerator>
+        <name>USING_DEFAULT_CORES</name>
+        <value>0</value>
+    </enumerator>
+    <enumerator>
+        <name>USING_NORMAL_CORES</name>
+        <value>1</value>
+    </enumerator>
+    <enumerator>
+        <name>USING_FUSED_CORES</name>
+        <value>2</value>
+    </enumerator>
+</enumerationType>
+
+<attribute>
+    <id>FUSED_CORE_OPTION</id>
+    <description>
+        If not loading PHYP or OPAL, then use this to
+        decide whether to use FUSED cores or NOT.
+    </description>
+    <simpleType>
+        <uint8_t>
+        </uint8_t>
+    </simpleType>
+    <persistency>volatile-zeroed</persistency>
+    <readable/>
+    <writeable/>
+</attribute>
+
 </attributes>
diff --git a/src/usr/targeting/common/xmltohb/target_types.xml b/src/usr/targeting/common/xmltohb/target_types.xml
index ab45e631e..1bb969172 100755
--- a/src/usr/targeting/common/xmltohb/target_types.xml
+++ b/src/usr/targeting/common/xmltohb/target_types.xml
@@ -311,6 +311,8 @@
     <attribute><id>SBE_MASTER_INTR_SERVICE_DELAY_US</id></attribute>
     <attribute><id>CLEAR_DIMM_SPD_ENABLE</id></attribute>
     <attribute><id>MRW_MCS_PREFETCH_RETRY_THRESHOLD</id></attribute>
+    <!-- AVP override for fused cores or normal cores -->
+    <attribute><id>FUSED_CORE_OPTION</id></attribute>
 </targetType>
 
 <targetType>
@@ -1987,6 +1989,8 @@
     <attribute><id>DO_ABUS_DECONFIG</id></attribute>
     <attribute><id>PLCK_IPL_ATTR_OVERRIDES_EXIST</id></attribute>
     <attribute><id>PAYLOAD_IN_MIRROR_MEM</id></attribute>
+    <!-- AVP override for fused cores or normal cores -->
+    <attribute><id>FUSED_CORE_OPTION</id></attribute>
 </targetType>
 
 <!-- enc-node-power9 -->
diff --git a/src/usr/targeting/common/xmltohb/vbu_NAPLES.system.xml b/src/usr/targeting/common/xmltohb/vbu_NAPLES.system.xml
index 14263aa79..aee1b5e93 100644
--- a/src/usr/targeting/common/xmltohb/vbu_NAPLES.system.xml
+++ b/src/usr/targeting/common/xmltohb/vbu_NAPLES.system.xml
@@ -299,6 +299,7 @@
         <id>MRW_MBA_CACHELINE_INTERLEAVE_MODE_CONTROL</id>
         <default>1</default>
     </attribute>
+
 </targetInstance>
 
 <!-- System node 0 -->