WOF: More Phase 2 Fixes

    Key_Cronus_Test=PM_REGRESS

Change-Id: Ib2abc5ee3814e2cfc9be2d229293b98996379bb3
Original-Change-Id: Iba2ff6ffc0c2b06b3d2df1c40f707da63eb020b0
CQ: SW404912
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/46595
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Cronus HW CI <cronushw-ci+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

4 files changed, 333 insertions, 171 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
index c0f8b80e..c4c3d7d7 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
@@ -127,6 +127,8 @@ void p9_cme_pstate_intercme_in0_handler()
         out32_sh(CME_LCL_EIMR_OR, (BITS64SH(34, 2)));//Disable PMCR0/1
         g_eimr_override |= BITS64(34, 2);
 
+        //PGPE will update the LMCR[0] before sending the STOP PSTATE Doorbell.
+        //Here we update the PMSR to indicate that Pstates are no longer honored accordingly.
         p9_cme_pstate_pmsr_updt(G_cme_record.core_enabled);
 
         //Set Core GPMMR RESET_STATE_INDICATOR bit to show pstates have stopped
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
index c1353990..a5081bfc 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
@@ -50,6 +50,8 @@ extern GlobalPstateParmBlock* G_gppb;
 extern uint32_t G_ext_vrm_inc_rate_mult_usperus;
 extern uint32_t G_ext_vrm_dec_rate_mult_usperus;
 extern PgpePstateRecord G_pgpe_pstate_record;
+extern void p9_pgpe_ipc_ack_sgpe_ctrl_stop_updt_core_enable(ipc_msg_t* msg, void* arg);
+extern void p9_pgpe_ipc_ack_sgpe_ctrl_stop_updt_core_disable(ipc_msg_t* msg, void* arg);
 
 //
 //Global Data
@@ -160,19 +162,20 @@ void p9_pgpe_pstate_setup_process_pcb_type4()
             {
                 if (ccsr.value & CORE_MASK(c))
                 {
-                    G_pgpe_pstate_record.activeCores |= CORE_MASK(c);
+                    G_pgpe_pstate_record.activeDB |= CORE_MASK(c);
                 }
             }
         }
     }
 
     //Fill OCC Shared Memory Area fields
+    G_pgpe_pstate_record.activeCores = G_pgpe_pstate_record.activeDB;
     G_pgpe_pstate_record.pQuadState0->fields.active_cores = (G_pgpe_pstate_record.activeCores >> 16);
     G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_pgpe_pstate_record.activeCores & 0xFF00);
     G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads =  G_pgpe_pstate_record.activeQuads;
 
-    PK_TRACE_INF("PCB4: ActiveQuads=0x%x ActiveCores=0x%x\n", G_pgpe_pstate_record.activeQuads ,
-                 G_pgpe_pstate_record.activeCores);
+    PK_TRACE_INF("PCB4: ActiveQuads=0x%x ActiveDB=0x%x, ActiveCores=0x%x\n", G_pgpe_pstate_record.activeQuads ,
+                 G_pgpe_pstate_record.activeDB, G_pgpe_pstate_record.activeCores);
 
     //Enable PCB_INTR_TYPE4
     out32(OCB_OPIT4PRA_CLR, opit4pr);
@@ -204,7 +207,11 @@ void p9_pgpe_pstate_do_auction()
 
             for (c = FIRST_CORE_FROM_QUAD(q); c < LAST_CORE_FROM_QUAD(q); c++)
             {
-                if (G_pgpe_pstate_record.activeCores & CORE_MASK(c))
+                //If WOF_Enabled, only then activeCores is updated and represent the accurate state of activeCores.
+                //Otherwise, activeDB represents which cores are active.
+                //In the latter case, STOP 11 Entry/Exit can modify which cores are active
+                if ((G_pgpe_pstate_record.wofStatus == WOF_ENABLED && (G_pgpe_pstate_record.activeCores & CORE_MASK(c))) ||
+                    (G_pgpe_pstate_record.activeDB & CORE_MASK(c)))
                 {
                     if (G_pgpe_pstate_record.quadPSComputed[q] > G_pgpe_pstate_record.coresPSRequest[c])
                     {
@@ -260,13 +267,13 @@ void p9_pgpe_pstate_apply_clips()
         if (G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
         {
 
-            if (G_pgpe_pstate_record.wofEnabled == 1)
+            if (G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
             {
                 if (G_pgpe_pstate_record.wofClip > minPS && (G_pgpe_pstate_record.wofClip < maxPS))
                 {
                     minPS = G_pgpe_pstate_record.wofClip;
                 }
-                else if (G_pgpe_pstate_record.wofClip > maxPS)
+                else if (G_pgpe_pstate_record.wofClip >= maxPS)
                 {
                     minPS = maxPS;
                 }
@@ -339,7 +346,14 @@ void p9_pgpe_pstate_calc_wof()
     //Currently, PGPE only support VRATIO Fixed and VRATIO active cores only
     if (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_VRATIO)
     {
-        G_pgpe_pstate_record.vratio = (G_pgpe_pstate_record.numActiveCores << 16) / G_pgpe_pstate_record.numConfCores;
+        if (G_pgpe_pstate_record.numActiveCores != 0)
+        {
+            G_pgpe_pstate_record.vratio = ((G_pgpe_pstate_record.numActiveCores << 16) - 1) / G_pgpe_pstate_record.numConfCores;
+        }
+        else
+        {
+            G_pgpe_pstate_record.vratio = 0;
+        }
     }
     else
     {
@@ -347,13 +361,13 @@ void p9_pgpe_pstate_calc_wof()
     }
 
     //3. VFRT table lookup
-    G_pgpe_pstate_record.vindex = (G_pgpe_pstate_record.vratio - 0xA8B) / 0xAAC;
+    G_pgpe_pstate_record.vindex = (G_pgpe_pstate_record.vratio + 0xAAC - 0xA8B) / 0xAAC;
 
     //4. Update wofClip(int. variable)
     G_pgpe_pstate_record.wofClip =
         G_pgpe_pstate_record.pVFRT->vfrt_data[G_pgpe_pstate_record.findex][G_pgpe_pstate_record.vindex];
 
-    PK_TRACE_DBG("WFC: FClip_PS=0x%x, vindex=%d, vratio=%d", G_pgpe_pstate_record.wofClip, G_pgpe_pstate_record.vindex,
+    PK_TRACE_DBG("WFC: FClip_PS=0x%x, vindex=0x%x, vratio=0x%x", G_pgpe_pstate_record.wofClip, G_pgpe_pstate_record.vindex,
                  G_pgpe_pstate_record.vratio);
     p9_pgpe_pstate_apply_clips();
     G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.vratio << 16) |
@@ -376,7 +390,7 @@ void p9_pgpe_pstate_update_wof_state()
     wof_state->fields.vclip_mv = G_pgpe_pstate_record.eVidCurr;
     wof_state->fields.fratio = G_pgpe_pstate_record.fratio;
     wof_state->fields.vratio = G_pgpe_pstate_record.vratio;
-    PK_TRACE_DBG("WFU: FClip_PS=0x%x, vindex=%d, vratio=%d", G_pgpe_pstate_record.wofClip, G_pgpe_pstate_record.vindex,
+    PK_TRACE_DBG("WFC: FClip_PS=0x%x, vindex=0x%x, vratio=0x%x", G_pgpe_pstate_record.wofClip, G_pgpe_pstate_record.vindex,
                  G_pgpe_pstate_record.vratio);
 }
 
@@ -452,7 +466,7 @@ void p9_pgpe_send_db0(uint64_t db0, uint32_t coreVector, uint32_t unicast, uint3
     else
     {
 #if NIMBUS_DD_LEVEL == 10
-        p9_dd1_db_multicast_wr(PCB_MULTICAST_GRP1 | CPPM_CMEDB0, db0, G_pgpe_pstate_record.activeCores);
+        p9_dd1_db_multicast_wr(PCB_MULTICAST_GRP1 | CPPM_CMEDB0, db0, coreVector);
 #else
         GPE_PUTSCOM(PCB_MULTICAST_GRP1 | CPPM_CMEDB0, db0)
 #endif
@@ -460,7 +474,7 @@ void p9_pgpe_send_db0(uint64_t db0, uint32_t coreVector, uint32_t unicast, uint3
 
     if (ack == PGPE_DB0_ACK_WAIT_CME)
     {
-        p9_pgpe_wait_cme_db_ack(ackVector);//Wait for ACKs from all QuadManagers
+        p9_pgpe_wait_cme_db_ack(ackVector);//Wait for ACKs from QuadManagers
     }
 }
 
@@ -525,7 +539,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
     for (q = 0; q < MAX_QUADS; q++)
     {
         //Exclude deconfigured quads
-        if (qcsr.fields.ex_config & (0xC00 >> (q << 1)))
+        if (qcsr.fields.ex_config & (QUAD_EX0_MASK(q) | QUAD_EX1_MASK(q)))
         {
             GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_STAT, q), dpll.value);
 
@@ -627,7 +641,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
             //Give Quad Manager CME control of DPLL through inter-ppm
             GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_OR, q), BIT64(26));
 
-            if (qcsr.fields.ex_config &  (0x800 >> (q << 1)))
+            if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
             {
                 //CME Scratch0
                 GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
@@ -636,7 +650,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
             }
 
-            if (qcsr.fields.ex_config &  (0x400 >> (q << 1)))
+            if (qcsr.fields.ex_config &  QUAD_EX1_MASK(q))
             {
                 //CME Scratch0
                 GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 1), value);
@@ -679,7 +693,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
     db0_clip_bcast.fields.quad4_clip = G_pgpe_pstate_record.psClipMax[4];
     db0_clip_bcast.fields.quad5_clip = G_pgpe_pstate_record.psClipMax[5];
     p9_pgpe_send_db0(db0_clip_bcast.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_UNICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
@@ -694,7 +708,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
     db0_clip_bcast.fields.quad4_clip = G_pgpe_pstate_record.psClipMin[4];
     db0_clip_bcast.fields.quad5_clip = G_pgpe_pstate_record.psClipMin[5];
     p9_pgpe_send_db0(db0_clip_bcast.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_UNICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
@@ -712,7 +726,7 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
     db0.fields.quad5_ps = G_pgpe_pstate_record.quadPSTarget[5];
 
     p9_pgpe_send_db0(db0.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_UNICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
@@ -791,7 +805,7 @@ void p9_pgpe_pstate_set_pmcr_owner(uint32_t owner)
         if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
         {
             //CME0 within this quad
-            if (qcsr.fields.ex_config & (0x800 >> (q << 1)))
+            if (qcsr.fields.ex_config & (0x800 >> (q * 2)))
             {
                 if (owner == PMCR_OWNER_HOST)
                 {
@@ -804,7 +818,7 @@ void p9_pgpe_pstate_set_pmcr_owner(uint32_t owner)
             }
 
             //CME1 within this quad
-            if (qcsr.fields.ex_config & (0x400 >> (q << 1)))
+            if (qcsr.fields.ex_config & (0x400 >> (q * 2)))
             {
                 if (owner == PMCR_OWNER_HOST)
                 {
@@ -841,13 +855,13 @@ void p9_pgpe_pstate_stop()
         if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
         {
             //CME0 within this quad
-            if (qcsr.fields.ex_config & (0x800 >> (q << 1)))
+            if (qcsr.fields.ex_config & (0x800 >> (q * 2)))
             {
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 0), BIT64(0));
             }
 
             //CME1 within this quad
-            if (qcsr.fields.ex_config & (0x400 >> (q << 1)))
+            if (qcsr.fields.ex_config & (0x400 >> (q * 2)))
             {
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 1), BIT64(0));
             }
@@ -855,16 +869,26 @@ void p9_pgpe_pstate_stop()
     }
 
     p9_pgpe_send_db0(db0_stop.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_UNICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
 
+    //Set status in OCC_Scratch2
     uint32_t occScr2 = in32(OCB_OCCS2);
     occScr2 &= ~BIT32(PGPE_PSTATE_PROTOCOL_ACTIVE);
     out32(OCB_OCCS2, occScr2);
 
     G_pgpe_pstate_record.pstatesStatus = PSTATE_STOPPED;
+
+    //ACK back to OCC
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+    ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+    args->msg_cb.rc = PGPE_RC_SUCCESS;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+    ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+
     PK_TRACE_INF("PSS: Stop Done");
 }
 
@@ -901,7 +925,7 @@ void p9_pgpe_pstate_clip_bcast(uint32_t clip_bcast_type)
     }
 
     p9_pgpe_send_db0(db0.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_UNICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
@@ -929,18 +953,18 @@ void p9_pgpe_pstate_process_quad_entry_notify(uint32_t quadsRequested)
         {
             GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26)); //Open DPLL for SCOMs
 
-            G_pgpe_pstate_record.activeCores &= ~(QUAD_ALL_CORES_MASK(q));
+            G_pgpe_pstate_record.activeDB &= ~(QUAD_ALL_CORES_MASK(q));
             out32(OCB_OPIT4PRA_CLR, QUAD_ALL_CORES_MASK(q)); //Clear any pending PCB_Type4
 
             //CME_Scratch0[DB0_PROCESSING_ENABLE]=0
-            if (qcsr.fields.ex_config &  (0x800 >> (q << 1)))
+            if (qcsr.fields.ex_config &  (0x800 >> (q * 2)))
             {
                 GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
                 value &= ~BIT64(CME_SCRATCH_DB0_PROCESSING_ENABLE);
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
             }
 
-            if (qcsr.fields.ex_config &  (0x400 >> (q << 1)))
+            if (qcsr.fields.ex_config &  (0x400 >> (q * 2)))
             {
                 GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 1), value);
                 value &= ~BIT64(CME_SCRATCH_DB0_PROCESSING_ENABLE);
@@ -977,12 +1001,10 @@ void p9_pgpe_pstate_process_quad_entry_done(uint32_t quadsRequested)
 {
     //Update Shared Memory Region
     G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads &= (~(quadsRequested));
-    G_pgpe_pstate_record.pQuadState0->fields.active_cores = (G_pgpe_pstate_record.activeCores >> 16);
-    G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_pgpe_pstate_record.activeCores & 0xFF00);
     PK_TRACE_INF("QE: (Done), Vec=0x%x\n", quadsRequested);
 
     //If WOF Enabled, then interlock with OCC
-    if(G_pgpe_pstate_record.wofEnabled == 1)
+    if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
     {
         GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
     }
@@ -1041,6 +1063,97 @@ void p9_pgpe_pstate_process_quad_exit(uint32_t quadsRequested)
 }
 
 //
+// p9_pgpe_pstate_wof_ctrl
+//
+void p9_pgpe_pstate_wof_ctrl(uint32_t action, uint32_t activeCores, uint32_t activeQuads)
+{
+    uint32_t c;
+
+    if (action == PGPE_ACTION_WOF_ON)
+    {
+        G_pgpe_pstate_record.wofStatus = WOF_ENABLED;
+        //Set to value returned by SGPE or initial value determined during boot(equal to configured cores)
+        G_pgpe_pstate_record.activeCores = activeCores << 8;
+
+        //Update Shared Memory Region
+        G_pgpe_pstate_record.pQuadState0->fields.active_cores = G_pgpe_pstate_record.activeCores >> 16;
+        G_pgpe_pstate_record.pQuadState1->fields.active_cores |= (G_pgpe_pstate_record.activeCores & 0xFF00);
+        G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = activeQuads;
+
+        G_pgpe_pstate_record.numActiveCores = 0;
+
+        for (c = 0; c < MAX_CORES; c++)
+        {
+            if (G_pgpe_pstate_record.activeCores & CORE_MASK(c))
+            {
+                G_pgpe_pstate_record.numActiveCores += 1;
+            }
+        }
+
+        PK_TRACE_DBG("WCT: numActiveCores=0x%x", G_pgpe_pstate_record.numActiveCores);
+        PK_TRACE_DBG("WCT: activeCores=0x%x", G_pgpe_pstate_record.activeCores);
+
+        p9_pgpe_pstate_calc_wof();
+        PK_TRACE_DBG("WCT: WOF Enabled");
+    }
+    else if (action == PGPE_ACTION_WOF_OFF)
+    {
+        G_pgpe_pstate_record.wofStatus = WOF_DISABLED;
+        p9_pgpe_pstate_update_wof_state();
+
+        //ACK back pending WOF
+        ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].cmd;
+        ipcmsg_wof_control_t* args = (ipcmsg_wof_control_t*)async_cmd->cmd_data;
+        args->msg_cb.rc = PGPE_RC_SUCCESS;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack = 0;
+        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].cmd, IPC_RC_SUCCESS);
+
+        p9_pgpe_optrace(ACK_WOF_CTRL);
+        PK_TRACE_DBG("WCT: WOF Disabled");
+    }
+}
+
+//
+//p9_pgpe_pstate_send_ctrl_stop_updt
+//
+void p9_pgpe_pstate_send_ctrl_stop_updt(uint32_t action)
+{
+    uint32_t rc;
+
+#if SGPE_IPC_ENABLED == 1
+    G_sgpe_control_updt.fields.return_code = 0x0;
+    G_sgpe_control_updt.fields.action = action;
+    G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
+
+    if (action == CTRL_STOP_UPDT_ENABLE_CORE)
+    {
+        //Send "Enable Core Stop Updates" IPC to SGPE
+        ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
+                     IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
+                     p9_pgpe_ipc_ack_sgpe_ctrl_stop_updt_core_enable,
+                     NULL);
+        PK_TRACE_DBG("PTH: Sent CTRL_STOP_UPDT(ENABLE_CORE) to SGPE");
+    }
+    else
+    {
+        ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
+                     IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
+                     p9_pgpe_ipc_ack_sgpe_ctrl_stop_updt_core_disable,
+                     NULL);
+        PK_TRACE_DBG("PTH: Sent CTRL_STOP_UPDT(DISABLE_CORE) to SGPE");
+    }
+
+    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
+
+    if(rc)
+    {
+        PK_PANIC(PGPE_SGPE_IPC_SEND_BAD_RC);
+    }
+
+#endif// _SGPE_IPC_ENABLED_
+}
+
+//
 // p9_pgpe_pstate_apply_safe_clips
 //
 void p9_pgpe_pstate_apply_safe_clips()
@@ -1098,7 +1211,7 @@ void p9_pgpe_pstate_safe_mode()
     G_pgpe_pstate_record.pstatesStatus = suspend ? PSTATE_PM_SUSPEND_PENDING : PSTATE_SAFE_MODE;
 
     //Mark WOF Disabled so that PGPE doesn't interlock with OCC anymore
-    G_pgpe_pstate_record.wofEnabled = 0;
+    G_pgpe_pstate_record.wofStatus = WOF_DISABLED;
 
     //Operation Trace Entry
     trace = suspend ? ACK_PM_SUSP : ACK_SAFE_DONE;
@@ -1140,13 +1253,13 @@ void p9_pgpe_pstate_send_suspend_stop()
         if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
         {
             //CME0 within this quad
-            if (qcsr.fields.ex_config & (0x800 >> (q << 1)))
+            if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
             {
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 0), BIT64(0));
             }
 
             //CME1 within this quad
-            if (qcsr.fields.ex_config & (0x400 >> (q << 1)))
+            if (qcsr.fields.ex_config & QUAD_EX1_MASK(q))
             {
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 1), BIT64(0));
             }
@@ -1243,6 +1356,8 @@ void p9_pgpe_pstate_do_step()
             G_pgpe_pstate_record.eVidNext = targetEVid;
             G_pgpe_pstate_record.globalPSNext = G_pgpe_pstate_record.globalPSTarget;
 
+            PK_TRACE_DBG("STEP: <= step_size");
+
             for (q = 0; q < MAX_QUADS; q++)
             {
                 if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
@@ -1284,6 +1399,7 @@ void p9_pgpe_pstate_do_step()
 
         }
 
+        PK_TRACE_DBG("STEP: eVidNext=0x%x, glbPSNext=0x%x", G_pgpe_pstate_record.eVidCurr, G_pgpe_pstate_record.globalPSNext);
         p9_pgpe_pstate_freq_updt();
         p9_pgpe_pstate_updt_ext_volt(targetEVid);
     }
@@ -1293,6 +1409,7 @@ void p9_pgpe_pstate_do_step()
 
         if ((targetEVid - G_pgpe_pstate_record.eVidCurr) <= G_gppb->ext_vrm_step_size_mv)
         {
+            PK_TRACE_DBG("STEP: <= step_size");
             G_pgpe_pstate_record.eVidNext = targetEVid;
             G_pgpe_pstate_record.globalPSNext = G_pgpe_pstate_record.globalPSTarget;
 
@@ -1306,6 +1423,7 @@ void p9_pgpe_pstate_do_step()
         }
         else
         {
+            PK_TRACE_DBG("STEP: > step_size");
             G_pgpe_pstate_record.eVidNext = G_pgpe_pstate_record.eVidCurr + G_gppb->ext_vrm_step_size_mv;
             G_pgpe_pstate_record.globalPSNext = p9_pgpe_gppb_intp_ps_from_ext_vdd(G_pgpe_pstate_record.eVidNext);
 
@@ -1336,6 +1454,7 @@ void p9_pgpe_pstate_do_step()
             }
         }
 
+        PK_TRACE_DBG("STEP: eVidNext=0x%x, glbPSNext=0x%x", G_pgpe_pstate_record.eVidCurr, G_pgpe_pstate_record.globalPSNext);
         p9_pgpe_pstate_updt_ext_volt(targetEVid);
         p9_pgpe_pstate_freq_updt();
     }
@@ -1476,7 +1595,7 @@ void p9_pgpe_pstate_freq_updt()
 
     p9_pgpe_optrace(ACTL_BROADCAST);
     p9_pgpe_send_db0(db0.value,
-                     G_pgpe_pstate_record.activeCores,
+                     G_pgpe_pstate_record.activeDB,
                      PGPE_DB0_MULTICAST,
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
index 7f1f9092..67a29571 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
@@ -82,8 +82,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
         pk_irq_sub_critical_exit(&ctx);
     }
 
-
-    // Set OCC Scratch2[PGPE_ACTIVE], so that external world knows that PGPE is UP
     occScr2 = in32(OCB_OCCS2);
     occScr2 |= BIT32(PGPE_ACTIVE);
     out32(OCB_OCCS2, occScr2);
@@ -129,6 +127,11 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                 (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 1) ||
                 (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack == 1))
             {
+                //Enter sub-critical section
+                //Process any pending ACKs in sub-critical section. Otherwise, it's possible that the inrange check
+                //below is interrupted, and new clips are calculated
+                pk_irq_sub_critical_enter(&ctx);
+
                 inRange = 1;
                 uint32_t minPS;
 
@@ -138,12 +141,19 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                     {
                         minPS = G_pgpe_pstate_record.psClipMin[q];
 
-                        if (G_pgpe_pstate_record.wofEnabled)
+                        if (G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
                         {
+                            //If WOF_CLIP is between thermal clips, then pull down(higher value) mininum pstate
                             if ((G_pgpe_pstate_record.wofClip <= G_pgpe_pstate_record.psClipMax[q]) &&
                                 (minPS < G_pgpe_pstate_record.wofClip))
                             {
                                 minPS = G_pgpe_pstate_record.wofClip;
+                                //If WOF_CLIP is lower(higher value) than the thermal max clips, then pull down(higher value)
+                                //for mininum pstate to thermal max clip
+                            }
+                            else if (G_pgpe_pstate_record.wofClip >= G_pgpe_pstate_record.psClipMax[q])
+                            {
+                                minPS = G_pgpe_pstate_record.psClipMax[q];
                             }
                         }
 
@@ -160,7 +170,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                 {
                     if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_ack == 1)
                     {
-                        pk_irq_sub_critical_enter(&ctx);
 
                         //Notify CMEs about Updated Pmin and Pmax
                         if (G_pgpe_pstate_record.pendingPminClipBcast)
@@ -175,7 +184,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                             G_pgpe_pstate_record.pendingPmaxClipBcast = 0;
                         }
 
-                        pk_irq_sub_critical_exit(&ctx);
 
                         ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].cmd;
                         ipcmsg_clip_update_t* args = (ipcmsg_clip_update_t*)async_cmd->cmd_data;
@@ -183,6 +191,7 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_ack = 0;
                         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].cmd, IPC_RC_SUCCESS);
                         p9_pgpe_optrace(ACK_CLIP_UPDT);
+                        restore_irq = 1;
                     }
 
                     if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_ack == 1)
@@ -212,9 +221,12 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                                 p9_pgpe_optrace(ACK_QUAD_ACTV);
                             }
                         }
+
+                        restore_irq = 1;
                     }
 
-                    if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 1)
+                    if ((G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 1) &&
+                        (G_pgpe_pstate_record.wofStatus == WOF_ENABLED))
                     {
                         p9_pgpe_pstate_update_wof_state();
                         ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].cmd;
@@ -230,15 +242,17 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                         p9_pgpe_pstate_update_wof_state();
                         ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd;
                         ipcmsg_s2p_update_active_cores_t* args = (ipcmsg_s2p_update_active_cores_t*)async_cmd->cmd_data;
+                        args->fields.return_active_cores = G_pgpe_pstate_record.activeCores >> 8;
                         args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
-                        args->fields.return_active_cores = G_pgpe_pstate_record.activeCores;
                         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
                         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd, IPC_RC_SUCCESS);
                         p9_pgpe_optrace(ACK_CORES_ACTV);
+                        restore_irq = 1;
                     }
 
-                    restore_irq = 1;
                 }
+
+                pk_irq_sub_critical_exit(&ctx); //Exit sub-critical section
             } //Pending ACKs
 
             //Check if IPC should be opened again
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
index 40bb2e1c..36b9157d 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
@@ -40,8 +40,6 @@ extern TraceData_t G_pgpe_optrace_data;
 extern PgpePstateRecord G_pgpe_pstate_record;
 extern PgpeHeader_t* G_pgpe_header_data;
 extern GlobalPstateParmBlock* G_gppb;
-GPE_BUFFER(extern ipcmsg_p2s_ctrl_stop_updates_t G_sgpe_control_updt);
-extern ipc_async_cmd_t G_ipc_msg_pgpe_sgpe;
 
 //
 //Local Function Prototypes
@@ -54,6 +52,8 @@ void p9_pgpe_process_wof_ctrl();
 void p9_pgpe_process_wof_vfrt();
 void p9_pgpe_process_set_pmcr_req();
 void p9_pgpe_process_registration();
+void p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_enable();
+void p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_disable();
 
 //
 //Process Request Thread
@@ -72,6 +72,19 @@ void p9_pgpe_thread_process_requests(void* arg)
     asm volatile ("tw 0, 31, 0");
 #endif
 
+    // Set OCC Scratch2[PGPE_ACTIVE], so that external world knows that PGPE is UP
+    // 00 - WOF Phase1(Vratio Fixed,Fratio Fixed)
+    // 01 - WOF Phase2(Vratio Active Cores, Fratio Fixed)
+    // 10 - WOF Phase3(Vratio Full, Fratio Fixed)
+    // 11 - WOF Phase4(Vratio Full, Fratio Full)
+    // Currently, only Phase 1 and 2 are supported.
+    if ((G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_VRATIO) ||
+        (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_VRATIO_MODIFIER) ||
+        (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_FRATIO))
+    {
+        out32(OCB_OCCFLG_OR, BIT32(29));
+    }
+
     PK_TRACE_INF("PTH:Inited");
 
     while(1)
@@ -90,6 +103,30 @@ void p9_pgpe_thread_process_requests(void* arg)
         PK_TRACE_DBG("PTH: Process Task");
 
         //Go through IPC Pending Table
+
+        //We must process this before CORES and QUADS active updt. Once, OCC has sent
+        //the WOF_DISABLE, PGPE must stop interlocking quads active updt with it. Also, PGPE
+        //must properly ensure that any pending CORE active update are processed properly
+        //once WOF_DISABLE has been received. SGPE should get all ACKS, but no more WOF
+        //calculation be performed
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_processing)
+        {
+            p9_pgpe_process_wof_ctrl();
+        }
+
+        //We must process this "before" ACTIVE_CORES_UPDT  as SGPE might have sent one after
+        //sending ACK for CTRL_UPDT_STOP[Core Enable], and we must have marked wofState = Enabled
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_ENABLE].pending_processing)
+        {
+            p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_enable();
+
+            if(G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_ENABLE].pending_ack == 1)
+            {
+                restore_irq = 0;
+            }
+        }
+
+
         if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing == 1)
         {
             p9_pgpe_process_sgpe_updt_active_cores();
@@ -100,6 +137,18 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
+        //We must process this "after" ACTIVE_CORES_UPDT to make sure any pending ACTIVE_CORES_UPDT
+        //have ACKed back to SGPE
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_DISABLE].pending_processing)
+        {
+            p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_disable();
+
+            if(G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_DISABLE].pending_ack == 1)
+            {
+                restore_irq = 0;
+            }
+        }
+
         if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing == 1)
         {
             p9_pgpe_process_sgpe_updt_active_quads();
@@ -120,16 +169,6 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_processing)
-        {
-            p9_pgpe_process_wof_ctrl();
-
-            if(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 1)
-            {
-                restore_irq = 0;
-            }
-        }
-
         if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_processing == 1)
         {
             p9_pgpe_process_wof_vfrt();
@@ -190,7 +229,7 @@ void p9_pgpe_process_sgpe_updt_active_cores()
 
     G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing = 0;
 
-    if(G_pgpe_pstate_record.wofEnabled == 0)
+    if(G_pgpe_pstate_record.wofStatus != WOF_ENABLED)
     {
         PK_TRACE_DBG("PTH: C Updt(WOF_Disabled)");
         args->fields.return_code = PGPE_WOF_RC_NOT_ENABLED;
@@ -199,24 +238,47 @@ void p9_pgpe_process_sgpe_updt_active_cores()
     }
     else
     {
+        //Store separately as shared SRAM location is split
+        PK_TRACE_DBG("PTH: Core Updt type=%u(0/1=EN/EX) reqCores=0x%x", args->fields.update_type, args->fields.active_cores);
+
+        //If ENTRY type then send ACK to SGPE immediately
+        //Otherwise, wait to ACK until WOF Clip has been applied(from actuate_pstate thread)
+        if (args->fields.update_type == UPDATE_ACTIVE_CORES_TYPE_ENTRY)
+        {
+            G_pgpe_pstate_record.activeCores &= ~(args->fields.active_cores << 8);
+            args->fields.return_active_cores = G_pgpe_pstate_record.activeCores >> 8;
+            args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
+        }
+        else
+        {
+            G_pgpe_pstate_record.activeCores |= (args->fields.active_cores << 8);
+            ack_now = 0;
+        }
+
         //Update Shared Memory Region
-        G_pgpe_pstate_record.pQuadState0->fields.active_cores = (args->fields.active_cores >> 8);
-        G_pgpe_pstate_record.pQuadState1->fields.active_cores = (args->fields.active_cores & 0xFF) << 8;
+        G_pgpe_pstate_record.pQuadState0->fields.active_cores = (G_pgpe_pstate_record.activeCores >> 16);
+        G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_pgpe_pstate_record.activeCores & 0x0000FF00);
 
-        PK_TRACE_DBG("PTH: Core Updt type=%u activeCores=0x%x)", args->fields.update_type, args->fields.active_cores);
+        PK_TRACE_DBG("PTH: numActiveCores=0x%x,activeCores=0x%x", G_pgpe_pstate_record.numActiveCores,
+                     G_pgpe_pstate_record.activeCores);
+        PK_TRACE_DBG("PTH: quadPS2=0x%08x%08x,activeCores=0x%x", G_pgpe_pstate_record.pQuadState0->value >> 32,
+                     G_pgpe_pstate_record.pQuadState0->value, G_pgpe_pstate_record.pQuadState0->fields.active_cores );
+        PK_TRACE_DBG("PTH: quadPS1=0x%08x%08x,activeCores=0x%x", G_pgpe_pstate_record.pQuadState1->value >> 32,
+                     G_pgpe_pstate_record.pQuadState1->value, G_pgpe_pstate_record.pQuadState1->fields.active_cores);
 
-        //Store separately as shared SRAM location is split
-        G_pgpe_pstate_record.activeCores = args->fields.active_cores;
+        //Calculate number of active cores
         G_pgpe_pstate_record.numActiveCores = 0;
 
         for (c = 0; c < MAX_CORES; c++)
         {
-            if (c & G_pgpe_pstate_record.activeCores)
+            if (G_pgpe_pstate_record.activeCores  & CORE_MASK(c))
             {
-                G_pgpe_pstate_record.numActiveCores += 1;
+                G_pgpe_pstate_record.numActiveCores++;
             }
         }
 
+        PK_TRACE_DBG("PTH: numActiveCores=0x%x", G_pgpe_pstate_record.numActiveCores);
+
         //OP_TRACE(Do before auction and wof calculation)
         G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.activeQuads << 24)  |
                                       ((args->fields.update_type == UPDATE_ACTIVE_CORES_TYPE_ENTRY) ? 0x2000000 : 0x1000000) |
@@ -226,25 +288,13 @@ void p9_pgpe_process_sgpe_updt_active_cores()
         //Do auction and wof calculation
         p9_pgpe_pstate_do_auction();
         p9_pgpe_pstate_calc_wof();
-
-        //If ENTRY type then send ACK to SGPE immediately
-        //Otherwise, wait to ACK until WOF Clip has been applied(from actuate_pstate thread)
-        if (args->fields.update_type == UPDATE_ACTIVE_CORES_TYPE_ENTRY)
-        {
-            PK_TRACE_DBG("PTH: Core Entry ACK back to SGPE");
-            args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
-            PK_TRACE_DBG("PTH: Core Updt ENTRY ACKed");
-        }
-        else
-        {
-            ack_now = 0;
-        }
     }
 
     if (ack_now == 1)
     {
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd, IPC_RC_SUCCESS);
+        PK_TRACE_DBG("PTH: Core Entry ACK back to SGPE");
         p9_pgpe_optrace(ACK_CORES_ACTV);
     }
 
@@ -295,9 +345,10 @@ void p9_pgpe_process_sgpe_updt_active_quads()
         G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (args->fields.requested_quads << 2);
 
         //WOF Enabled
-        if(G_pgpe_pstate_record.wofEnabled == 1)
+        if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
         {
-            GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
+            PK_TRACE_DBG("PTH: OCCLFG[30] set");
+            out32(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
             ack_now = 0;
         }
         else
@@ -552,54 +603,20 @@ void p9_pgpe_process_wof_ctrl()
         //If WOF ON
         if (args->action == PGPE_ACTION_WOF_ON)
         {
-            if(G_pgpe_pstate_record.wofEnabled == 0)
+            if(G_pgpe_pstate_record.wofStatus == WOF_DISABLED)
             {
                 PK_TRACE_DBG("PTH: WOF Ctrl=ON,WOF_Enabled=0");
 
                 if ((G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_VRATIO) ||
                     (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_VRATIO_MODIFIER))
                 {
-#if SGPE_IPC_ENABLED == 1
-                    uint32_t rc;
-                    PkMachineContext  ctx;
-                    //Send "Disable Core Stop Updates" IPC to SGPE
-                    G_sgpe_control_updt.fields.return_code = 0x0;
-                    G_sgpe_control_updt.fields.action = CTRL_STOP_UPDT_ENABLE_CORE;
-                    G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
-                    ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
-                                 IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
-                                 p9_pgpe_pstate_ipc_rsp_cb_sem_post,
-                                 (void*)&G_pgpe_pstate_record.sem_sgpe_wait);
-
-                    //send the command
-                    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
-
-                    if(rc)
-                    {
-                        PGPE_PANIC_AND_TRACE(PGPE_SGPE_IPC_SEND_BAD_RC);
-                    }
-
-                    //Wait for SGPE ACK with alive Quads
-                    pk_irq_vec_restore(&ctx);
-                    pk_semaphore_pend(&(G_pgpe_pstate_record.sem_actuate), PK_WAIT_FOREVER);
-
-                    if (G_sgpe_control_updt.fields.return_code == IPC_SGPE_PGPE_RC_SUCCESS)
-                    {
-                        //Update Shared Memory Region
-                        G_pgpe_pstate_record.pQuadState0->fields.active_cores = G_sgpe_control_updt.fields.active_cores >> 8;
-                        G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_sgpe_control_updt.fields.active_cores & 0xFF) << 8;
-                        G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = G_sgpe_control_updt.fields.active_quads;
-                    }
-                    else
-                    {
-                        PGPE_PANIC_AND_TRACE(PGPE_SGPE_CTRL_STOP_UPDT_BAD_ACK);
-                    }
-
-#endif// _SGPE_IPC_ENABLED_
+                    p9_pgpe_pstate_send_ctrl_stop_updt(CTRL_STOP_UPDT_ENABLE_CORE);
+                }
+                else
+                {
+                    p9_pgpe_pstate_wof_ctrl(PGPE_ACTION_WOF_ON, G_pgpe_pstate_record.activeDB >> 8, G_pgpe_pstate_record.activeQuads);
                 }
 
-                G_pgpe_pstate_record.wofEnabled = 1;
-                p9_pgpe_pstate_calc_wof();
                 ack_now = 0;
                 G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.activeQuads << 24) |
                                               (args->action << 16) |
@@ -616,56 +633,25 @@ void p9_pgpe_process_wof_ctrl()
         }
         else if (args->action == PGPE_ACTION_WOF_OFF)
         {
-            if(G_pgpe_pstate_record.wofEnabled == 1)
+            if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
             {
                 PK_TRACE_DBG("PTH: WOF Ctrl=OFF,WOF_Enabled=1");
 
                 if ((G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_VRATIO) ||
                     (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_VRATIO_MODIFIER))
                 {
-#if SGPE_IPC_ENABLED == 1
-                    uint32_t rc;
-                    PkMachineContext  ctx;
-                    //Send "Disable Core Stop Updates" IPC to SGPE
-                    G_sgpe_control_updt.fields.return_code = 0x0;
-                    G_sgpe_control_updt.fields.action = CTRL_STOP_UPDT_DISABLE_CORE;
-                    G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
-                    ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
-                                 IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
-                                 p9_pgpe_pstate_ipc_rsp_cb_sem_post,
-                                 (void*)&G_pgpe_pstate_record.sem_sgpe_wait);
-
-                    //send the command
-                    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
-
-                    if(rc)
-                    {
-                        PGPE_PANIC_AND_TRACE(PGPE_SGPE_IPC_SEND_BAD_RC);
-                    }
-
-                    //Wait for SGPE ACK with alive Quads
-                    pk_irq_vec_restore(&ctx);
-                    pk_semaphore_pend(&(G_pgpe_pstate_record.sem_actuate), PK_WAIT_FOREVER);
-
-                    if (G_sgpe_control_updt.fields.return_code == SGPE_PGPE_IPC_RC_SUCCESS)
-                    {
-                        //Update Shared Memory Region
-                        G_pgpe_pstate_record.pQuadState0->fields.active_cores = G_sgpe_control_updt.fields.active_cores >> 8;
-                        G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_sgpe_control_updt.fields.active_cores & 0xFF) << 8;
-                        G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = G_sgpe_control_updt.fields.active_quads;
-                    }
-                    else
-                    {
-                        PGPE_PANIC_AND_TRACE(PGPE_SGPE_CTRL_STOP_UPDT_BAD_ACK);
-                    }
+                    p9_pgpe_pstate_send_ctrl_stop_updt(CTRL_STOP_UPDT_DISABLE_CORE);
+                }
+                else
+                {
+                    p9_pgpe_pstate_wof_ctrl(PGPE_ACTION_WOF_OFF, G_pgpe_pstate_record.activeDB >> 8, G_pgpe_pstate_record.activeQuads);
                 }
 
-#endif// _SGPE_IPC_ENABLED_
-
-                G_pgpe_pstate_record.wofEnabled = 0;
                 G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.activeQuads << 24) |
-                                              (args->action << 16) | (in32(OCB_QCSR) >> 16);
+                                              (args->action << 16) |
+                                              (in32(OCB_QCSR) >> 16);
                 p9_pgpe_optrace(PRC_WOF_CTRL);
+                ack_now = 0;
             }
             else
             {
@@ -735,7 +721,7 @@ void p9_pgpe_process_wof_vfrt()
 
         //Update VFRT pointer
         G_pgpe_pstate_record.pVFRT = args->homer_vfrt_ptr;
-        PK_TRACE_INF("PTH: VFRT Table");
+        PK_TRACE_INF("PTH: VFRT Table 0x%x", (uint32_t)G_pgpe_pstate_record.pVFRT);
         PK_TRACE_INF("Mgc=0x%x, ver=%d, vdnid=%x,VddQAId=%x", G_pgpe_pstate_record.pVFRT->vfrtHeader.magic_number,
                      G_pgpe_pstate_record.pVFRT->vfrtHeader.type_version,
                      G_pgpe_pstate_record.pVFRT->vfrtHeader.res_vdnId,
@@ -748,7 +734,7 @@ void p9_pgpe_process_wof_vfrt()
                                        G_pgpe_pstate_record.pVFRT->vfrtHeader.rsvd_QAId);
         p9_pgpe_optrace(PRC_WOF_VFRT);
 
-        if(G_pgpe_pstate_record.wofEnabled == 1)
+        if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED)
         {
             p9_pgpe_pstate_calc_wof();
             ack_now = 0;
@@ -865,7 +851,7 @@ void p9_pgpe_process_registration()
     ocb_qcsr_t qcsr;
     qcsr.value = in32(OCB_QCSR);
     volatile uint32_t opit4pr;
-    uint32_t opit4prQuad, q, c, oldActiveCores, oldActiveQuads, unicastCoresVector = 0, quadsRegisterProcess;
+    uint32_t opit4prQuad, q, c, oldActiveDB, oldActiveQuads, unicastCoresVector = 0, quadsRegisterProcess;
     uint32_t quadAckExpect = 0;
     uint64_t value;
     pgpe_db0_start_ps_bcast_t db0_glb_bcast;
@@ -873,7 +859,7 @@ void p9_pgpe_process_registration()
 
     //Save it for global bcast sync in case GlobalPSTarget is higher in
     //after doing auctions with just registered quads
-    oldActiveCores = G_pgpe_pstate_record.activeCores;
+    oldActiveDB = G_pgpe_pstate_record.activeDB;
     oldActiveQuads = G_pgpe_pstate_record.activeQuads;
 
     //Save it because if pstates are active, then we need to start pstates on quads
@@ -891,7 +877,7 @@ void p9_pgpe_process_registration()
             {
                 if (ccsr.value & CORE_MASK(c))
                 {
-                    G_pgpe_pstate_record.activeCores |= CORE_MASK(c);
+                    G_pgpe_pstate_record.activeDB |= CORE_MASK(c);
                 }
             }
 
@@ -901,7 +887,7 @@ void p9_pgpe_process_registration()
             G_pgpe_pstate_record.pendQuadsRegisterProcess &= ~QUAD_MASK(q);
 
             PK_TRACE_DBG("PTH: Quad %d Registered. qActive=0x%x cActive=0x%x", q, G_pgpe_pstate_record.activeQuads,
-                         G_pgpe_pstate_record.activeCores);
+                         G_pgpe_pstate_record.activeDB);
 
             G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.activeQuads << 24) | (G_pgpe_pstate_record.globalPSCurr << 16)
                                           | (in32(OCB_QCSR) >> 16);
@@ -957,7 +943,7 @@ void p9_pgpe_process_registration()
             db0_glb_bcast.fields.quad5_ps = G_pgpe_pstate_record.quadPSNext[5];
 
             p9_pgpe_send_db0(db0_glb_bcast.value,
-                             oldActiveCores,
+                             oldActiveDB,
                              PGPE_DB0_MULTICAST,
                              PGPE_DB0_ACK_WAIT_CME,
                              oldActiveQuads);
@@ -969,7 +955,7 @@ void p9_pgpe_process_registration()
             if(quadsRegisterProcess & QUAD_MASK(q))
             {
                 //Write CME_SCRATCH and PMSR0/1 registers
-                if (qcsr.fields.ex_config &  (0x800 >> (q << 1)))
+                if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
                 {
                     //CME_Scratch
                     GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
@@ -978,7 +964,7 @@ void p9_pgpe_process_registration()
                     GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
                 }
 
-                if (qcsr.fields.ex_config &  (0x400 >> (q << 1)))
+                if (qcsr.fields.ex_config & QUAD_EX1_MASK(q))
                 {
                     //CME_Scratch
                     GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 1), value);
@@ -993,7 +979,7 @@ void p9_pgpe_process_registration()
 
                 for (c = FIRST_CORE_FROM_QUAD(q); c < LAST_CORE_FROM_QUAD(q); c++)
                 {
-                    if (G_pgpe_pstate_record.activeCores & CORE_MASK(c))
+                    if (G_pgpe_pstate_record.activeDB & CORE_MASK(c))
                     {
                         unicastCoresVector |= CORE_MASK(c);
                     }
@@ -1069,9 +1055,50 @@ void p9_pgpe_process_registration()
         }
     }
 
-    //Update Active Cores in OCC Shared SRAM
-    G_pgpe_pstate_record.pQuadState0->fields.active_cores = (G_pgpe_pstate_record.activeCores >> 16);
-    G_pgpe_pstate_record.pQuadState1->fields.active_cores = (G_pgpe_pstate_record.activeCores & 0xFF00);
-
     PK_TRACE_DBG("PTH: Register Exit");
 }
+
+//
+//p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_enable
+//
+void p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_enable()
+{
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_ENABLE].cmd;
+    ipcmsg_p2s_ctrl_stop_updates_t* args = (ipcmsg_p2s_ctrl_stop_updates_t*)async_cmd->cmd_data;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_ENABLE].pending_processing = 0;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_ENABLE].pending_ack = 0;
+
+    PK_TRACE_DBG("IPC: SGPE ACKed CTRL_STOP_UPDT(ENABLE_CORE)");
+
+    if (args->fields.return_code == IPC_SGPE_PGPE_RC_SUCCESS)
+    {
+        p9_pgpe_pstate_wof_ctrl(PGPE_ACTION_WOF_ON, args->fields.active_cores, args->fields.active_quads);
+    }
+    else
+    {
+        PK_PANIC(PGPE_SGPE_CTRL_STOP_UPDT_BAD_ACK);
+    }
+}
+
+//
+//p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_disable
+//
+void p9_pgpe_process_ack_sgpe_ctrl_stop_updt_core_disable()
+{
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_DISABLE].cmd;
+    ipcmsg_p2s_ctrl_stop_updates_t* args = (ipcmsg_p2s_ctrl_stop_updates_t*)async_cmd->cmd_data;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_DISABLE].pending_processing = 0;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_CORE_DISABLE].pending_ack = 0;
+
+    PK_TRACE_DBG("IPC: SGPE ACKed CTRL_STOP_UPDT(DISABLE_CORE)");
+
+    if (args->fields.return_code == SGPE_PGPE_IPC_RC_SUCCESS)
+    {
+        p9_pgpe_pstate_wof_ctrl(PGPE_ACTION_WOF_OFF, args->fields.active_cores, args->fields.active_quads);
+
+    }
+    else
+    {
+        PK_PANIC(PGPE_SGPE_CTRL_STOP_UPDT_BAD_ACK);
+    }
+}