Integrate HWPs for istep 16

Change-Id: Ic8bac9940b00eae01c175f51dd1872b2bb128b95
Depends-on:Ib0db5ac2b9b5d5f5c2967ff97794493d867fb04b
RTC:133832
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/20552
Tested-by: Jenkins Server
Reviewed-by: Andrew J. Geissler <andrewg@us.ibm.com>
Tested-by: FSP CI Jenkins
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

6 files changed, 224 insertions, 479 deletions

diff --git a/src/usr/isteps/istep16/call_host_activate_master.C b/src/usr/isteps/istep16/call_host_activate_master.C
index 2050cf163..ad2d57961 100644
--- a/src/usr/isteps/istep16/call_host_activate_master.C
+++ b/src/usr/isteps/istep16/call_host_activate_master.C
@@ -23,46 +23,41 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 
-#include <errl/errlentry.H>
-#include    <stdint.h>
-#include    <errno.h>
-#include    <config.h>
-#include    <initservice/initserviceif.H>
-#include    <trace/interface.H>
-#include    <initservice/taskargs.H>
-#include    <errl/errlentry.H>
 
+// Error Handling
+#include    <errl/errlentry.H>
+#include    <errl/errlmanager.H>
 #include    <initservice/isteps_trace.H>
-#include    <initservice/istepdispatcherif.H>
-
 #include    <isteps/hwpisteperror.H>
 #include    <errl/errludtarget.H>
-
 #include    <intr/interrupt.H>
 #include    <console/consoleif.H>
 
 //  targeting support
-#include    <targeting/common/commontargeting.H>
-#include    <targeting/common/utilFilter.H>
 #include    <targeting/namedtarget.H>
 #include    <targeting/attrsync.H>
-#include    <runtime/runtime.H>
+#include    <fapi2/target.H>
 
-#include    <sys/task.h>
+//SBE interfacing
+#include    <sbeio/sbeioif.H>
 #include    <sys/misc.h>
 
-#include <util/misc.H>
+//Import directory (EKB)
+#include    <p9_block_wakeup_intr.H>
+#include    <p9_cpu_special_wakeup.H>
+
+//HWP invoker
+#include    <fapi2/plat_hwp_invoker.H>
 
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   ISTEP;
 using   namespace   ISTEP_ERROR;
+using   namespace   p9specialWakeup;
 
 
 namespace ISTEP_16
 {
-    //@TODO RTC:133832 call p9_sbe_trigger_stop15.C HWP
-    //@TODO RTC:133832 call p9_block_wakeup_intr.C HWP
 void* call_host_activate_master (void *io_pArgs)
 {
     IStepError  l_stepError;
@@ -72,11 +67,7 @@ void* call_host_activate_master (void *io_pArgs)
 
     errlHndl_t  l_errl  =   NULL;
 
-    // @@@@@    CUSTOM BLOCK:   @@@@@
-
     do  {
-
-#if 0
         // find the master core, i.e. the one we are running on
         TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "call_host_activate_master: Find master core: " );
@@ -84,48 +75,38 @@ void* call_host_activate_master (void *io_pArgs)
         const TARGETING::Target*  l_masterCore  = getMasterCore( );
         assert( l_masterCore != NULL );
 
-        TARGETING::Target* l_cpu_target = const_cast<TARGETING::Target *>
+        TARGETING::Target* l_core_target = const_cast<TARGETING::Target *>
                                           ( getParentChip( l_masterCore ) );
 
-        //@TODO RTC:133832
-        // Cast OUR type of target to a FAPI type of target.
-        //const fapi::Target l_fapi_cpu_target( TARGET_TYPE_PROC_CHIP,
-        //                   (const_cast<TARGETING::Target*> (l_cpu_target)) );
-
-        // Pass in Master EX target
-        const TARGETING::Target* l_masterEx = getExChiplet(l_masterCore);
-        assert(l_masterEx != NULL );
+        // Cast OUR type of target to a FAPI2 type of target.
+        const fapi2::Target<fapi2::TARGET_TYPE_CORE> l_fapi2_coreTarget(
+                                const_cast<TARGETING::Target*> (l_core_target));
 
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                   "call_host_activate_master: call proc_prep_master_winkle. "
+                   "call_host_activate_master: About to start deadman loop... "
                    "Target HUID %.8X",
-                    TARGETING::get_huid(l_masterEx));
-
-        //@TODO RTC:133832
-        // cast OUR type of target to a FAPI type of target.
-        //const fapi::Target l_fapi_ex_target( TARGET_TYPE_EX_CHIPLET,
-        //                   (const_cast<TARGETING::Target*> (l_masterEx)) );
-
-        //  call the HWP with each fapi::Target
-        //FAPI_INVOKE_HWP( l_errl,
-        //                 proc_prep_master_winkle,
-        //                 l_fapi_ex_target,
-        //                 true  );
+                    TARGETING::get_huid(l_fapi2_coreTarget));
+
+// @TODO RTC:147553 Enable startDeadmanLoop
+//In the future possibly move default "waitTime" value to SBEIO code
+//          uint64_t waitTime = 10000;
+//          l_errl = SBEIO::startDeadmanLoop(waitTime);
+
         if ( l_errl )
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-            "proc_prep_master_winkle ERROR : Returning errorlog, reason=0x%x",
+            "startDeadmanLoop ERROR : Returning errorlog, reason=0x%x",
                 l_errl->reasonCode() );
 
             // capture the target data in the elog
-            ErrlUserDetailsTarget(l_masterEx).addToLog( l_errl );
+            ErrlUserDetailsTarget(l_core_target).addToLog( l_errl );
 
             break;
         }
         else
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "proc_prep_master_winkle SUCCESS"  );
+                       "startDeadManLoop SUCCESS"  );
         }
 
         //Because of a bug in how the SBE injects the IPI used to wake
@@ -140,63 +121,60 @@ void* call_host_activate_master (void *io_pArgs)
                        "call_host_activate_master ERROR : MBOX::suspend");
             break;
         }
+           //@TODO RTC:137564 Support for thread-specific interrupts
+//         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "draining interrupt Q");
+//         INTR::drainQueue();
 
-        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "draining interrupt Q");
-        INTR::drainQueue();
 
-
-        // Call p8_block_wakeup_intr to prevent stray interrupts from
+        // Call p9_block_wakeup_intr to prevent stray interrupts from
         // popping core out of winkle before SBE sees it.
 
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                   "call_host_activated_master: call p8_block_wakeup_intr(SET) "
+                   "call_host_activated_master: call p9_block_wakeup_intr(SET) "
                    "Target HUID %.8x",
-                   TARGETING::get_huid(l_masterEx) );
+                   TARGETING::get_huid(l_fapi2_coreTarget) );
 
-        //@TODO RTC:133832
-        //FAPI_INVOKE_HWP( l_errl,
-        //                 p8_block_wakeup_intr,
-        //                 l_fapi_ex_target,
-        //                 BLKWKUP_SET );
+        FAPI_INVOKE_HWP( l_errl,
+                        p9_block_wakeup_intr,
+                        l_fapi2_coreTarget,
+                        p9pmblockwkup::SET );
 
         if ( l_errl )
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-            "p8_block_wakeup_intr ERROR : Returning errorlog, reason=0x%x",
+            "p9_block_wakeup_intr ERROR : Returning errorlog, reason=0x%x",
                 l_errl->reasonCode() );
 
             // capture the target data in the elog
-            ErrlUserDetailsTarget(l_masterEx).addToLog( l_errl );
+            ErrlUserDetailsTarget(l_core_target).addToLog( l_errl );
 
             break;
         }
         else
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "p8_block_wakeup_intr SUCCESS"  );
+                       "p9_block_wakeup_intr SUCCESS"  );
         }
 
         // Clear special wakeup
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Disable special wakeup on master core");
 
-        //@TODO RTC:133832
-        /*
-        FAPI_INVOKE_HWP(l_errl, p8_cpu_special_wakeup,
-                        l_fapi_ex_target,
+        FAPI_INVOKE_HWP(l_errl, p9_cpu_special_wakeup,
+                        l_fapi2_coreTarget,
                         SPCWKUP_DISABLE,
                         HOST);
-        */
+
 
         if(l_errl)
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-            "Disable p8_cpu_special_wakeup ERROR : Returning errorlog,"
+            "Disable p9_cpu_special_wakeup ERROR : Returning errorlog,"
             " reason=0x%x",
                 l_errl->reasonCode() );
 
             // capture the target data in the elog
-            ErrlUserDetailsTarget(l_masterEx).addToLog( l_errl );
+            ErrlUserDetailsTarget(l_core_target).addToLog( l_errl );
 
             break;
         }
@@ -205,7 +183,6 @@ void* call_host_activate_master (void *io_pArgs)
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
                       "Disable special wakeup on master core SUCCESS");
         }
-#endif
 
 
         //  put the master into winkle.
@@ -247,7 +224,6 @@ void* call_host_activate_master (void *io_pArgs)
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Returned from Winkle." );
 
-#if (0)
         //Re-enable the mailbox
         l_errl = MBOX::resume();
         if (l_errl)
@@ -259,57 +235,49 @@ void* call_host_activate_master (void *io_pArgs)
 
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Call proc_stop_deadman_timer. Target %.8X",
-                    TARGETING::get_huid(l_cpu_target) );
-
-        //  call the HWP with each fapi::Target
-        bool l_sbeIntrServiceActive = false;
-        //@TODO RTC:133832
-/*        FAPI_INVOKE_HWP( l_errl,
-                         proc_stop_deadman_timer,
-                         l_fapi_cpu_target,
-                         l_sbeIntrServiceActive  );
-*/
+                    TARGETING::get_huid(l_core_target) );
+
+// @TODO RTC:147553 Enable stopDeadmanLoop
+//         l_errl = SBEIO::stopDeadmanLoop();
+
         if ( l_errl )
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "proc_stop_deadman_timer ERROR : "
+                       "stopDeadmanLoop ERROR : "
                        "Returning errorlog, reason=0x%x",
                        l_errl->reasonCode() );
 
             // capture the target data in the elog
-            ErrlUserDetailsTarget(l_cpu_target).addToLog( l_errl );
+            ErrlUserDetailsTarget(l_core_target).addToLog( l_errl );
 
             break;
         }
         else
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                       "proc_prep_master_winkle SUCCESS"  );
+                       "stopDeadmanLoop SUCCESS"  );
         }
         TARGETING::Target* sys = NULL;
         TARGETING::targetService().getTopLevelTarget(sys);
-        sys->setAttr<ATTR_SBE_MASTER_INTR_SERVICE_ENABLED>
-                                                    (l_sbeIntrServiceActive);
 
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                    "Enable special wakeup on master core");
 
-        //@TODO RTC:133832
-        /*
-        FAPI_INVOKE_HWP(l_errl, p8_cpu_special_wakeup,
-                        l_fapi_ex_target,
+
+        FAPI_INVOKE_HWP(l_errl, p9_cpu_special_wakeup,
+                        l_fapi2_coreTarget,
                         SPCWKUP_ENABLE,
                         HOST);
-*/
+
         if(l_errl)
         {
             TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-            "Enable p8_cpu_special_wakeup ERROR : Returning errorlog, "
+            "Enable p9_cpu_special_wakeup ERROR : Returning errorlog, "
             "reason=0x%x",
                 l_errl->reasonCode() );
 
             // capture the target data in the elog
-            ErrlUserDetailsTarget(l_masterEx).addToLog( l_errl );
+            ErrlUserDetailsTarget(l_core_target).addToLog( l_errl );
 
             break;
         }
@@ -318,11 +286,9 @@ void* call_host_activate_master (void *io_pArgs)
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
                       "Enable special wakeup on master core SUCCESS");
         }
-#endif
 
     }   while ( 0 );
 
-    // @@@@@    END CUSTOM BLOCK:   @@@@@
     if( l_errl )
     {
         // Create IStep error log and cross reference error that occurred
diff --git a/src/usr/isteps/istep16/call_host_activate_slave_cores.C b/src/usr/isteps/istep16/call_host_activate_slave_cores.C
index b7018a86f..6e4977524 100644
--- a/src/usr/isteps/istep16/call_host_activate_slave_cores.C
+++ b/src/usr/isteps/istep16/call_host_activate_slave_cores.C
@@ -23,18 +23,9 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 
-#include <errl/errlentry.H>
-#include    <stdint.h>
-#include    <errno.h>
-#include    <config.h>
-#include    <initservice/initserviceif.H>
-#include    <trace/interface.H>
-#include    <initservice/taskargs.H>
 #include    <errl/errlentry.H>
-
+#include    <errno.h>
 #include    <initservice/isteps_trace.H>
-#include    <initservice/istepdispatcherif.H>
-
 #include    <isteps/hwpisteperror.H>
 #include    <errl/errludtarget.H>
 
@@ -45,31 +36,31 @@
 #include    <targeting/common/commontargeting.H>
 #include    <targeting/common/utilFilter.H>
 #include    <targeting/namedtarget.H>
-#include    <targeting/attrsync.H>
-#include    <runtime/runtime.H>
-
+#include    <fapi2/target.H>
+#include    <errl/errlmanager.H>
 #include    <sys/task.h>
 #include    <sys/misc.h>
 
-#include <util/misc.H>
+#include    <fapi2/plat_hwp_invoker.H>
+#include    <p9_cpu_special_wakeup.H>
 
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   ISTEP;
 using   namespace   ISTEP_ERROR;
-
+using   namespace   p9specialWakeup;
 
 namespace ISTEP_16
 {
 void* call_host_activate_slave_cores (void *io_pArgs)
 {
     IStepError  l_stepError;
-#if 0
-    errlHndl_t  l_timeout_errl  =   NULL;
+
+//     errlHndl_t  l_timeout_errl  =   NULL;
     errlHndl_t  l_errl          =   NULL;
 
 
-    TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_slave_cores entry" );
 
     // @@@@@    CUSTOM BLOCK:   @@@@@
@@ -88,7 +79,7 @@ void* call_host_activate_slave_cores (void *io_pArgs)
 
         CHIP_UNIT_ATTR l_coreId =
                 (*l_core)->getAttr<TARGETING::ATTR_CHIP_UNIT>();
-        FABRIC_GROUP_ID_ATTR l_logicalNodeId =
+        FABRIC_GROUP_ID_ATTR l_logicalGroupId =
           l_processor->getAttr<TARGETING::ATTR_FABRIC_GROUP_ID>();
         FABRIC_CHIP_ID_ATTR l_chipId =
           l_processor->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
@@ -97,7 +88,10 @@ void* call_host_activate_slave_cores (void *io_pArgs)
         assert( sys != NULL );
         uint64_t en_threads = sys->getAttr<ATTR_ENABLED_THREADS>();
 
-        uint64_t pir = PIR_t(l_logicalNodeId, l_chipId, l_coreId).word;
+        const fapi2::Target<fapi2::TARGET_TYPE_CORE> l_fapi2_coreTarget(
+              const_cast<TARGETING::Target*> (*l_core));
+
+        uint64_t pir = PIR_t(l_logicalGroupId, l_chipId, l_coreId).word;
 
         if (pir != l_masterCoreID)
         {
@@ -105,237 +99,113 @@ void* call_host_activate_slave_cores (void *io_pArgs)
                        "call_host_activate_slave_cores: Waking %x",
                        pir );
 
-            // Get EX FAPI target
-            TARGETING::TargetHandleList targetList;
-            getParentAffinityTargets(targetList,
-                                     (*l_core),
-                                     TARGETING::CLASS_UNIT,
-                                     TARGETING::TYPE_EX);
-
+            int rc = cpu_start_core(pir,en_threads);
 
-            // verify the list has one entry, see SW272212.
-            if( targetList.size() == 1 )
+            // Handle time out error
+            if (-ETIME == rc)
             {
-                TARGETING::Target* l_ex = targetList[0];
-                //@TODO RTC:133832
-                //const fapi::Target l_fapi_ex_target( TARGET_TYPE_EX_CHIPLET,
-                //        const_cast<TARGETING::Target*>(l_ex) );
-
-                int rc = cpu_start_core(pir,en_threads);
-
-                // Handle time out error
-                if (-ETIME == rc)
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                            "call_host_activate_slave_cores: "
-                            "Time out rc from kernel %d on core %x",
-                            rc,
-                            pir);
-
-                    //@TODO RTC:133832
-                    //FAPI_INVOKE_HWP( l_timeout_errl, proc_check_slw_done,
-                    //l_fapi_ex_target);
-                    if (l_timeout_errl)
-                    {
-                        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                                "ERROR : proc_check_slw_done" );
-                        // Add chip target info
-                        ErrlUserDetailsTarget(l_processor).addToLog(
-                                                               l_timeout_errl );
-                        // Create IStep error log
-                        l_stepError.addErrorDetails(l_timeout_errl);
-                        // Commit error
-                        errlCommit( l_timeout_errl, HWPF_COMP_ID );
-                        break;
-                    }
-                    else
-                    {
-                        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                                "SUCCESS : proc_check_slw_done - "
-                                 "SLW is in clean state");
-                    }
-                }
-                // Create error log
-                if (0 != rc)
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                            "call_host_activate_slave_cores: "
-                            "Error from kernel %d on core %x",
-                            rc,
-                            pir);
-                    /*@
-                     * @errortype
-                     * @reasoncode  RC_BAD_RC
-                     * @severity    ERRORLOG::ERRL_SEV_UNRECOVERABLE
-                     * @moduleid    MOD_HOST_ACTIVATE_SLAVE_CORES
-                     * @userdata1   PIR of failing core.
-                     * @userdata2   rc of cpu_start_core().
-                     *
-                     * @devdesc Kernel returned error when trying to activate
-                     *          core.
-                     */
-                    errlHndl_t l_errl =
-                        new ERRORLOG::ErrlEntry(
-                                ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                MOD_HOST_ACTIVATE_SLAVE_CORES,
-                                RC_BAD_RC,
-                                pir,
-                                rc );
-
-                    // Callout core that failed to wake up.
-                    l_errl->addHwCallout(*l_core,
-                            HWAS::SRCI_PRIORITY_MED,
-                            HWAS::DECONFIG,
-                            HWAS::GARD_Predictive);
-
-                    l_stepError.addErrorDetails( l_errl );
-                    errlCommit( l_errl, HWPF_COMP_ID );
-                    break;
-                }
-                else //Core out of winkle sucessfully, issue SPWU for PRD
-                {
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                            "Running p8_cpu_special_wakeup (ENABLE)"
-                            " EX target HUID %.8X",
-                            TARGETING::get_huid(l_ex));
-
-                    // Enable special wakeup on core
-                    //@TODO RTC:133832
-                    /*FAPI_INVOKE_HWP( l_errl,
-                            p8_cpu_special_wakeup,
-                            l_fapi_ex_target,
-                            SPCWKUP_ENABLE,
-                            HOST);
-                    */
-                    if( l_errl )
-                    {
-                        ErrlUserDetailsTarget(l_ex).addToLog( l_errl );
-
-                        // Create IStep error log and cross ref error that
-                        // occurred
-                        l_stepError.addErrorDetails( l_errl );
-
-                        // Commit Error
-                        errlCommit( l_errl, HWPF_COMP_ID );
-
-                        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                                "ERROR : enable p8_cpu_special_wakeup, "
-                                "PLID=0x%x", l_errl->plid()  );
-                    }
-                    else
-                    {
-                        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                                "SUCCESS: enable p8_cpu_special_wakeup");
-                    }
-                }
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                        "call_host_activate_slave_cores: "
+                        "Time out rc from kernel %d on core %x",
+                        rc,
+                        pir);
+
+//@TODO RTC:147376
+//Spoke with Thi and he said this is not planned for awhile
+//FAPI_INVOKE_HWP( l_timeout_errl, proc_check_slw_done,
+//l_fapi2_ex_target);
+//                 if (l_timeout_errl)
+//                 {
+//                     TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+//                             "ERROR : proc_check_slw_done" );
+//                     // Add chip target info
+//                     ErrlUserDetailsTarget(l_processor).addToLog(
+//                                                             l_timeout_errl );
+//                     // Create IStep error log
+//                     l_stepError.addErrorDetails(l_timeout_errl);
+//                     // Commit error
+//                     errlCommit( l_timeout_errl, HWPF_COMP_ID );
+//                     break;
+//                 }
+//                 else
+//                 {
+//                     TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+//                             "SUCCESS : proc_check_slw_done - "
+//                               "SLW is in clean state");
+//                 }
             }
-            else
+            // Create error log
+            if (0 != rc)
             {
-                // wrong number of targets in the list, create an error log
                 TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                            "ERROR: call to getParentAffinityTarget "
-                            "returned %d instead of 1", targetList.size() );
+                        "call_host_activate_slave_cores: "
+                        "Error from kernel %d on core %x",
+                        rc,
+                        pir);
                 /*@
-                 * @errortype
-                 * @reasoncode  RC_INCORRECT_TARGET_COUNT
-                 * @severity    ERRORLOG::ERRL_SEV_UNRECOVERABLE
-                 * @moduleid    MOD_HOST_ACTIVATE_SLAVE_CORES
-                 * @userdata1   PIR of failing core.
-                 * @userdata2   number of targets returned
-                 *
-                 * @devdesc     Call to getParentAffinityTarget requesting
-                 *              the number of EX chips with parent affinity
-                 *              to a core, returned an incorrect vector size,
-                 *              the expected size is 1.
-                 *
-                 * @custdec     A problem occurred during the IPL of the system.
-                 *
-                 */
-                l_errl =
-                    new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                  * @errortype
+                  * @reasoncode  RC_BAD_RC
+                  * @severity    ERRORLOG::ERRL_SEV_UNRECOVERABLE
+                  * @moduleid    MOD_HOST_ACTIVATE_SLAVE_CORES
+                  * @userdata1   PIR of failing core.
+                  * @userdata2   rc of cpu_start_core().
+                  *
+                  * @devdesc Kernel returned error when trying to activate
+                  *          core.
+                  */
+                errlHndl_t l_errl =
+                    new ERRORLOG::ErrlEntry(
+                            ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                             MOD_HOST_ACTIVATE_SLAVE_CORES,
-                            RC_INCORRECT_TARGET_COUNT,
+                            RC_BAD_RC,
                             pir,
-                            targetList.size(), true);
-                // Create IStep error log and cross ref error that occurred
-                l_stepError.addErrorDetails( l_errl );
+                            rc );
 
-                // Commit Error
+                // Callout core that failed to wake up.
+                l_errl->addHwCallout(*l_core,
+                        HWAS::SRCI_PRIORITY_MED,
+                        HWAS::DECONFIG,
+                        HWAS::GARD_Predictive);
+
+                l_stepError.addErrorDetails( l_errl );
                 errlCommit( l_errl, HWPF_COMP_ID );
+                break;
             }
-        }
-    }
 
-    if( l_stepError.isNull() )
-    {
-        // Call proc_post_winkle
-        TARGETING::TargetHandleList l_procTargetList;
-        getAllChips(l_procTargetList, TYPE_PROC);
-
-        // Done activate all master/slave cores.
-        // Run post winkle check on all EX targets, one proc at a time.
-        for (TargetHandleList::const_iterator l_procIter =
-             l_procTargetList.begin();
-             l_procIter != l_procTargetList.end();
-             ++l_procIter)
-        {
-            const TARGETING::Target* l_pChipTarget = *l_procIter;
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "Running proc_post_winkle on chip HUID %.8X",
-                TARGETING::get_huid(l_pChipTarget));
+                    "Running p9_cpu_special_wakeup (ENABLE)"
+                    " EX target HUID %.8X",
+                    TARGETING::get_huid(*l_core));
+
+            // Enable special wakeup on core
+            FAPI_INVOKE_HWP( l_errl,
+                    p9_cpu_special_wakeup,
+                    l_fapi2_coreTarget,
+                    SPCWKUP_ENABLE,
+                    HOST);
+
+            if( l_errl )
+            {
+                ErrlUserDetailsTarget(*l_core).addToLog( l_errl );
 
-            // Get EX list under this proc
-            TARGETING::TargetHandleList l_exList;
-            getChildChiplets( l_exList, l_pChipTarget, TYPE_EX );
+                // Create IStep error log and cross ref error that
+                // occurred
+                l_stepError.addErrorDetails( l_errl );
 
-            for (TargetHandleList::const_iterator
-                l_exIter = l_exList.begin();
-                l_exIter != l_exList.end();
-                ++l_exIter)
-            {
-                const TARGETING::Target * l_exTarget = *l_exIter;
+                // Commit Error
+                errlCommit( l_errl, HWPF_COMP_ID );
 
                 TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "Running proc_post_winkle on EX target HUID %.8X",
-                TARGETING::get_huid(l_exTarget));
-
-                // cast OUR type of target to a FAPI type of target.
-                //@TODO RTC:133832
-                /*
-                fapi::Target l_fapi_ex_target( TARGET_TYPE_EX_CHIPLET,
-                         (const_cast<TARGETING::Target*>(l_exTarget)) );
-
-                //  call the HWP with each fapi::Target
-                FAPI_INVOKE_HWP( l_errl,
-                                 proc_post_winkle,
-                                 l_fapi_ex_target);
-                */
-                if ( l_errl )
-                {
-                    // capture the target data in the elog
-                    ErrlUserDetailsTarget(l_pChipTarget).addToLog( l_errl );
-
-                    // Create IStep error log and cross ref error that occurred
-                    l_stepError.addErrorDetails( l_errl );
-
-                    // Commit Error
-                    errlCommit( l_errl, HWPF_COMP_ID );
-
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                             "ERROR : proc_post_winkle, PLID=0x%x",
-                             l_errl->plid()  );
-                }
-                else
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                               "SUCCESS : proc_post_winkle" );
-                }
+                        "ERROR : enable p9_cpu_special_wakeup, "
+                        "PLID=0x%x", l_errl->plid()  );
             }
-
-        }   // end for
-
-    }   // end if
-
+            else
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                        "SUCCESS: enable p9_cpu_special_wakeup");
+            }
+        }
+    }
     // @@@@@    END CUSTOM BLOCK:   @@@@@
 
 #ifdef CONFIG_ENABLE_CHECKSTOP_ANALYSIS
@@ -362,7 +232,7 @@ void* call_host_activate_slave_cores (void *io_pArgs)
 
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                "call_host_activate_slave_cores exit" );
-#endif
+
     // end task, returning any errorlogs to IStepDisp
     return l_stepError.getErrorHandle();
 }
diff --git a/src/usr/isteps/istep16/call_host_ipl_complete.C b/src/usr/isteps/istep16/call_host_ipl_complete.C
index 232027a1f..675598e6c 100644
--- a/src/usr/isteps/istep16/call_host_ipl_complete.C
+++ b/src/usr/isteps/istep16/call_host_ipl_complete.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -35,14 +35,18 @@
 //  targeting support
 #include    <targeting/common/commontargeting.H>
 #include    <targeting/common/utilFilter.H>
-#include    <targeting/namedtarget.H>
+
+#include    <fapi2/target.H>
+#include    <fapi2/plat_hwp_invoker.H>
+
+#include   <p9_switch_rec_attn.H>
 
 
 using   namespace   ERRORLOG;
 using   namespace   TARGETING;
 using   namespace   ISTEP;
 using   namespace   ISTEP_ERROR;
-
+using   namespace   fapi2;
 
 namespace ISTEP_16
 {
@@ -55,25 +59,19 @@ void* call_host_ipl_complete (void *io_pArgs)
                "call_host_ipl_complete entry" );
     do
     {
-#if 0
+
         // We only need to run cfsim on the master Processor.
         TARGETING::Target * l_masterProc =   NULL;
         (void)TARGETING::targetService().
             masterProcChipTargetHandle( l_masterProc );
 
-        //@TODO RTC:133832
-        //const fapi::Target l_fapi_proc_target( TARGET_TYPE_PROC_CHIP,
-        //        ( const_cast<TARGETING::Target*>(l_masterProc) ) );
-
-        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "Running proc_switch_cfsim HWP on target HUID %.8X",
-                TARGETING::get_huid(l_masterProc) );
-
-
-        //  call proc_switch_cfsim
-    // TODO: RTC 64136 - Comment out to work around Centaur FSI scom issue
-    // during BU
-    // RTC 64136 is opened to undo this when in-band scoms are available.
+//@TODO RTC:144076 L1 HWPs for Centuar+Cumulus
+//const fapi::Target l_fapi_proc_target( TARGET_TYPE_PROC_CHIP,
+//        ( const_cast<TARGETING::Target*>(l_masterProc) ) );
+//  call proc_switch_cfsim
+// TODO: RTC 64136 - Comment out to work around Centaur FSI scom issue
+// during BU
+// RTC 64136 is opened to undo this when in-band scoms are available.
 #if 0
         FAPI_INVOKE_HWP(l_err, proc_switch_cfsim, l_fapi_proc_target,
                         true, // RESET
@@ -84,6 +82,13 @@ void* call_host_ipl_complete (void *io_pArgs)
                         true, // FENCE_FSI1
                         true); // FENCE_PIB_SW1
 #endif
+
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                "Running proc_switch_cfsim HWP on target HUID %.8X",
+                TARGETING::get_huid(l_masterProc) );
+
+
+
         if (l_err)
         {
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
@@ -109,7 +114,9 @@ void* call_host_ipl_complete (void *io_pArgs)
                        "SUCCESS: proc_switch_cfsim HWP( )" );
         }
 
-
+//@TODO RTC:144076 L1 HWPs for Centuar+Cumulus
+// Need p9_switch_rec_attn for mem_chips
+#if 0
         if ( INITSERVICE::spBaseServicesEnabled())
         {
             // For FSP based systems, do not route centaur
@@ -130,15 +137,13 @@ void* call_host_ipl_complete (void *io_pArgs)
                          "Running cen_switch_rec_attn HWP on target HUID %.8X",
                          TARGETING::get_huid(l_memChip) );
 
-                // @TODO RTC:133832 cast OUR type of target
-                // to a FAPI type of target.
-                /*
-                fapi::Target l_fapi_centaur_target( TARGET_TYPE_MEMBUF_CHIP,
-                                                    l_memChip );
-                FAPI_INVOKE_HWP( l_err,
-                                 cen_switch_rec_attn,
-                                 l_fapi_centaur_target );
-                                 */
+              const fapi2::Target<TARGET_TYPE_MEMBUF_CHIP> l_fap2_centTarget(
+                            const_cast<TARGETING::Target*> (l_memChip));
+//@TODO RTC:144076 L1 HWPs for Centuar+Cumulus
+//                 FAPI_INVOKE_HWP( l_err,
+//                                  p9_switch_rec_attn,
+//                                  l_fap2_centTarget );
+
                 if (l_err)
                 {
                     // log error for this centaur and continue
@@ -166,124 +171,9 @@ void* call_host_ipl_complete (void *io_pArgs)
                 }
             }   // endfor
 
-
-            //  Loop through all the mcs in the system
-            //  and run proc_switch_rec_attn
-            TARGETING::TargetHandleList l_mcsTargetList;
-            getAllChiplets(l_mcsTargetList, TYPE_MCS);
-
-            for ( TargetHandleList::iterator l_iter = l_mcsTargetList.begin();
-                l_iter != l_mcsTargetList.end();
-                ++l_iter )
-            {
-                TARGETING::Target * l_mcsChiplet  =   (*l_iter) ;
-
-                //  dump physical path to target
-                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                      "Running cen_switch_rec_attn HWP on target HUID %.8X",
-                       TARGETING::get_huid(l_mcsChiplet) );
-
-                //@TODO RTC:133832 cast OUR type of target
-                //to a FAPI type of target.
-                /*fapi::Target l_fapi_mcs_target( TARGET_TYPE_MCS_CHIPLET,
-                                                l_mcsChiplet );
-
-                FAPI_INVOKE_HWP( l_err,
-                                 proc_switch_rec_attn,
-                                 l_fapi_mcs_target );
-                */
-                if (l_err)
-                {
-                    // log error for this mcs and continue
-
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                              "ERROR 0x%.8X: proc_switch_rec_attn HWP( )",
-                              l_err->reasonCode() );
-
-                    // Add all the details for this proc
-                    ErrlUserDetailsTarget myDetails(l_mcsChiplet);
-
-                    // capture the target data in the elog
-                    myDetails.addToLog(l_err);
-
-                    // Create IStep error log and cross ref error that occurred
-                    l_stepError.addErrorDetails( l_err );
-
-                    // Commit Error
-                    errlCommit( l_err, HWPF_COMP_ID );
-                }
-                else
-                {
-                    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-                               "SUCCESS: proc_switch_rec_attn HWP( )" );
-                }
-
-            }   //  endfor
         } // end if ( INITSERVICE::spBaseServicesEnabled())
-
-#ifdef CONFIG_PCIE_HOTPLUG_CONTROLLER
-        //  Loop through all the procs in the system
-        //  and run proc_pcie_slot_power to  power on the hot plug controller
-        //  for Sapphire systems. PowerVM will turn on the hot plug
-        //  controller.
-        if (is_sapphire_load())
-        {
-
-            //  get a list of all the procs in the system
-            TARGETING::Target* l_pMasterProcTarget = NULL;
-            TARGETING::targetService().
-                       masterProcChipTargetHandle(l_pMasterProcTarget);
-            TARGETING::TargetHandleList l_procTargetList;
-            getAllChips(l_procTargetList, TYPE_PROC);
-
-            for (TargetHandleList::const_iterator
-                    l_proc_iter = l_procTargetList.begin();
-                    l_proc_iter != l_procTargetList.end();
-                    ++l_proc_iter)
-            {
-                //  make a local copy of the Processor target
-                TARGETING::Target* l_pProcTarget = *l_proc_iter;
-
-                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                    "target HUID %.8X",
-                    TARGETING::get_huid(l_pProcTarget));
-
-                //@TODO RTC:133832
-                /*
-                fapi::Target l_fapiProcTarget( fapi::TARGET_TYPE_PROC_CHIP,
-                                       l_pProcTarget    );
-
-                // Invoke the HWP
-                FAPI_INVOKE_HWP(l_err,
-                        proc_pcie_slot_power,
-                        l_fapiProcTarget,
-                        true  );  // turn on
-                */
-                if (l_err)
-                {
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                          "ERROR : proc_pcie_hotplug_control",
-                          " failed, returning errorlog" );
-
-                    // capture the target data in the elog
-                    ErrlUserDetailsTarget(l_pProcTarget).addToLog( l_err );
-
-                    // informational. Don't add to istep error or return error
-                    l_err->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
-
-                    // Commit error log
-                    errlCommit( l_err, HWPF_COMP_ID );
-                }
-                else
-                {
-                    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                      "SUCCESS : proc_pcie_hotplug_control",
-                      " completed ok");
-                }
-            }   // endfor
-        }
-#endif
 #endif
+
         // Sync attributes to Fsp
         l_err = syncAllAttributesToFsp();
 
diff --git a/src/usr/isteps/istep16/call_host_secure_rng.C b/src/usr/isteps/istep16/call_host_secure_rng.C
index 04ac58313..2a4df6da2 100644
--- a/src/usr/isteps/istep16/call_host_secure_rng.C
+++ b/src/usr/isteps/istep16/call_host_secure_rng.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -32,7 +32,8 @@ namespace ISTEP_16
 void* call_host_secure_rng (void *io_pArgs)
 {
     errlHndl_t l_err = NULL;
-    //@TODO RTC:133832 call p9_secure_rng.C HWP
+    //@TODO RTC:147388 call p9_secure_rng.C HWP
+    //Joe McGill not finished.
     return l_err;
 }
 
diff --git a/src/usr/isteps/istep16/call_mss_scrub.C b/src/usr/isteps/istep16/call_mss_scrub.C
index 51dd2d718..436a05313 100644
--- a/src/usr/isteps/istep16/call_mss_scrub.C
+++ b/src/usr/isteps/istep16/call_mss_scrub.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015                             */
+/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -43,7 +43,7 @@ namespace ISTEP_16
 void* call_mss_scrub (void *io_pArgs)
 {
     IStepError l_stepError;
-#if 0
+
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_mss_scrub entry" );
 
     // There are performance issues and some functional deficiencies
@@ -54,7 +54,11 @@ void* call_mss_scrub (void *io_pArgs)
         return NULL;
     }
 
-    errlHndl_t l_errl = PRDF::startScrub();
+    errlHndl_t l_errl = NULL;
+
+//TODO RTC:147390 Enable background scrubbing
+//     errlHndl_t l_errl = PRDF::startScrub();
+
     if ( NULL != l_errl )
     {
         TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
@@ -66,7 +70,6 @@ void* call_mss_scrub (void *io_pArgs)
     }
 
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_mss_scrub exit" );
-#endif
     // end task, returning any errorlogs to IStepDisp
     return l_stepError.getErrorHandle();
 }
diff --git a/src/usr/isteps/istep16/makefile b/src/usr/isteps/istep16/makefile
index 2d70d9be8..efc902fd4 100644
--- a/src/usr/isteps/istep16/makefile
+++ b/src/usr/isteps/istep16/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2015
+# Contributors Listed Below - COPYRIGHT 2015,2016
 # [+] International Business Machines Corp.
 #
 #
@@ -24,6 +24,15 @@
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../../../..
 MODULE = istep16
+PROCEDURES_PATH    = ${ROOTPATH}/src/import/chips/p9/procedures
+
+
+EXTRAINCDIR += ${PROCEDURES_PATH}/hwp/pm/
+EXTRAINCDIR += ${PROCEDURES_PATH}/hwp/perv/
+EXTRAINCDIR += ${ROOTPATH}/src/import/hwpf/fapi2/include/
+EXTRAINCDIR += ${ROOTPATH}/src/include/usr/fapi2/
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/common/include/
+EXTRAINCDIR += ${ROOTPATH}/src/import/chips/p9/utils/
 
 OBJS += call_host_activate_master.o
 OBJS += call_host_activate_slave_cores.o
@@ -31,6 +40,12 @@ OBJS += call_host_secure_rng.o
 OBJS += call_mss_scrub.o
 OBJS += call_host_ipl_complete.o
 
-EXTRAINCDIR += ${ROOTPATH}/src/include/usr/ecmddatabuffer/
 
+include ${ROOTPATH}/procedure.rules.mk
+include  ${PROCEDURES_PATH}/hwp/pm/p9_block_wakeup_intr.mk
+#TODO RTC:147383 Need to create a seperate module for cpu_special_wakeup
+include  ${PROCEDURES_PATH}/hwp/pm/p9_cpu_special_wakeup.mk
 include ${ROOTPATH}/config.mk
+
+
+VPATH += ${ROOTPATH}/src/import/chips/p9/procedures/hwp/pm/