PSTATE: PGPE-SGPE Interaction Updates

Change-Id: Ifcda5054779766eb2acc40a539b09ddf02b9ae63
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39682
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Juan R. Medina <jrmedina@us.ibm.com>
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

13 files changed, 983 insertions, 668 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
index e7b245f4..c75b6c27 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
@@ -62,6 +62,7 @@ cme_pstate_db_data_t G_db_thread_data;
 //Function Prototypes
 //
 inline void p9_cme_pstate_process_db0();
+inline void p9_cme_pstate_register();
 inline void p9_cme_pstate_db0_start(cppm_cmedb0_t dbData, uint32_t cme_flags);
 inline void p9_cme_pstate_db0_glb_bcast(cppm_cmedb0_t dbData, uint32_t cme_flags);
 inline void p9_cme_pstate_db0_suspend(cppm_cmedb0_t dbData, uint32_t cme_flags);
@@ -279,6 +280,9 @@ void p9_cme_pstate_db_thread(void* arg)
     //and won't run this thread past this point.
     if (G_cme_pstate_record.qmFlag)
     {
+        //Register with PGPE
+        p9_cme_pstate_register();
+
         pk_semaphore_create(&G_cme_pstate_record.sem[1], 0, 1);
 
         PK_TRACE("DB_TH: Inited\n");
@@ -344,6 +348,19 @@ inline void p9_cme_pstate_process_db0()
     PK_TRACE_INF("DB_TH: Process DB0 Exit\n");
 }
 
+
+inline void p9_cme_pstate_register()
+{
+    //Send type4(ack doorbell)
+    ppm_pig_t ppmPigData;
+    ppmPigData.value = 0;
+    ppmPigData.fields.req_intr_type = 4;
+    ppmPigData.fields.req_intr_payload =  MSGID_PCB_TYPE4_QUAD_MGR_AVAILABLE;
+    send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
+
+    PK_TRACE_INF("DB_TH: Register Msg Sent\n");
+}
+
 //
 //Doorbell0 Start
 //
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe.h
index 46f57516..9e6ab309 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe.h
@@ -62,6 +62,8 @@
 #define OCC_IPC_IMMEDIATE_RESP      0x0080
 #define WOF_IPC_IMMEDIATE_RESP      0x0040
 #define CCSR_CORE_CONFIG_MASK    0x80000000
+#define PSTATE_START_OCC_FLAG 0
+#define PSTATE_START_OCC_IPC  1
 
 
 /// PGPE PState
@@ -69,6 +71,7 @@ void p9_pgpe_irq_handler_occ_error(void* arg, PkIrqId irq);
 void p9_pgpe_irq_handler_sgpe_halt(void* arg, PkIrqId irq);
 void p9_pgpe_irq_handler_xstop_gpe2(void* arg, PkIrqId irq);
 void p9_pgpe_irq_handler_pcb_type1(void* arg, PkIrqId irq);
+void p9_pgpe_irq_handler_pcb_type4(void* arg, PkIrqId irq);
 void p9_pgpe_irq_handler_ipi2_lo(void* arg, PkIrqId irq);
 void p9_pgpe_thread_process_requests(void* arg);
 void p9_pgpe_thread_actuate_pstates(void* arg);
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_gppb.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_gppb.h
index ae603670..f17c908c 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_gppb.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_gppb.h
@@ -28,10 +28,6 @@
 #include "pk.h"
 #include "p9_pstates_pgpe.h"
 
-//#define REGION_POWERSAVE_NOMINAL    1
-//#define REGION_NOMINAL_TURBO        0
-//#define REGION_TURBO_ULTRA          2
-
 #define EVID_SLOPE_FP_SHIFT         13
 #define MAX_DPLL_VALUE              255
 
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_ipc_handlers.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_ipc_handlers.c
index dbb1417a..0f0048c7 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_ipc_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_ipc_handlers.c
@@ -76,11 +76,30 @@ void p9_pgpe_ipc_init()
 //
 void p9_pgpe_ipc_405_start_stop(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("START_STOP: Entry\n");
+    PK_TRACE_INF("START_STOP: Entry\n");
 
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)cmd;
     ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
 
+    if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack == 0)
+    {
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd = cmd;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 1;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 1;
+    }
+    else
+    {
+        args->msg_cb.rc = PGPE_RC_REQ_WHILE_PENDING_ACK;
+        ipc_send_rsp(cmd, IPC_RC_SUCCESS);
+    }
+
+    if (G_pgpe_pstate_record.alreadySemPosted == 0)
+    {
+        pk_semaphore_post(&G_pgpe_pstate_record.sem_process_req);
+        G_pgpe_pstate_record.alreadySemPosted  = 1;
+    }
+
+    /*
     if(G_pgpe_header_data->g_pgpe_qm_flags & OCC_IPC_IMMEDIATE_RESP)
     {
         PK_TRACE_DBG("START_STOP: Imm\n");
@@ -135,6 +154,9 @@ void p9_pgpe_ipc_405_start_stop(ipc_msg_t* cmd, void* arg)
                     //Post to Actuate Thread
                     pk_semaphore_post(&G_pgpe_pstate_record.sem_actuate);
                 }
+                //else if PSTATE_SUSPEND_WHILE_STOPPED_INIT
+                    //pending ack
+                    //pending processing
                 else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
                 {
                     //\todo
@@ -177,8 +199,9 @@ void p9_pgpe_ipc_405_start_stop(ipc_msg_t* cmd, void* arg)
             }
         }
     }
+    */
 
-    PK_TRACE_DBG("START_STOP: Exit\n");
+    PK_TRACE_INF("START_STOP: Exit\n");
 }
 
 //
@@ -188,7 +211,7 @@ void p9_pgpe_ipc_405_start_stop(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_405_clips(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("405_CLIPS: Entry\n");
+    PK_TRACE_INF("405_CLIPS: Entry\n");
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)cmd;
     ipcmsg_set_pmcr_t* args = (ipcmsg_set_pmcr_t*)async_cmd->cmd_data;
 
@@ -210,7 +233,7 @@ void p9_pgpe_ipc_405_clips(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("405_CLIPS: Exit\n");
+    PK_TRACE_INF("405_CLIPS: Exit\n");
 }
 
 //
@@ -220,7 +243,7 @@ void p9_pgpe_ipc_405_clips(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_405_set_pmcr(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("405_SET_PMCR: Entry\n");
+    PK_TRACE_INF("405_SET_PMCR: Entry\n");
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)cmd;
     ipcmsg_set_pmcr_t* args = (ipcmsg_set_pmcr_t*)async_cmd->cmd_data;
 
@@ -243,7 +266,7 @@ void p9_pgpe_ipc_405_set_pmcr(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("405_SET_PMCR: Exit\n");
+    PK_TRACE_INF("405_SET_PMCR: Exit\n");
 }
 
 //
@@ -253,7 +276,7 @@ void p9_pgpe_ipc_405_set_pmcr(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_405_wof_control(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("405_WOF_CTRL: Entry\n");
+    PK_TRACE_INF("405_WOF_CTRL: Entry\n");
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)cmd;
     ipcmsg_wof_control_t* args = (ipcmsg_wof_control_t*)async_cmd->cmd_data;
 
@@ -275,7 +298,7 @@ void p9_pgpe_ipc_405_wof_control(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("405_WOF_CTRL: Exit\n");
+    PK_TRACE_INF("405_WOF_CTRL: Exit\n");
 }
 
 //
@@ -285,7 +308,7 @@ void p9_pgpe_ipc_405_wof_control(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_405_wof_vfrt(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("405_WOF_VFRT: Entry\n");
+    PK_TRACE_INF("405_WOF_VFRT: Entry\n");
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)cmd;
     ipcmsg_wof_vfrt_t* args = (ipcmsg_wof_vfrt_t*)async_cmd->cmd_data;
 
@@ -307,7 +330,7 @@ void p9_pgpe_ipc_405_wof_vfrt(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("405_WOF_VFRT: Exit\n");
+    PK_TRACE_INF("405_WOF_VFRT: Exit\n");
 }
 
 //
@@ -317,7 +340,7 @@ void p9_pgpe_ipc_405_wof_vfrt(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_sgpe_suspend_pstates(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("SGPE_SUSPEND_PSTATES: Entry\n");
+    PK_TRACE_INF("SGPE_SUSPEND_PSTATES: Entry\n");
 
     if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_ack == 0 &&
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_processing == 0)
@@ -341,7 +364,7 @@ void p9_pgpe_ipc_sgpe_suspend_pstates(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("SGPE_SUSPEND_PSTATES: Exit\n");
+    PK_TRACE_INF("SGPE_SUSPEND_PSTATES: Exit\n");
 }
 
 //
@@ -351,7 +374,7 @@ void p9_pgpe_ipc_sgpe_suspend_pstates(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_sgpe_updt_active_cores(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("SGPE_UPDT_CORES: Entry\n");
+    PK_TRACE_INF("SGPE_UPDT_CORES: Entry\n");
 
     if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack == 0 &&
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing == 0)
@@ -375,7 +398,7 @@ void p9_pgpe_ipc_sgpe_updt_active_cores(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("SGPE_UPDT_CORES: Exit\n");
+    PK_TRACE_INF("SGPE_UPDT_CORES: Exit\n");
 }
 
 //
@@ -385,7 +408,7 @@ void p9_pgpe_ipc_sgpe_updt_active_cores(ipc_msg_t* cmd, void* arg)
 //
 void p9_pgpe_ipc_sgpe_updt_active_quads(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_DBG("SGPE_UPDT_QUADS: Entry\n");
+    PK_TRACE_INF("SGPE_UPDT_QUADS: Entry\n");
 
     if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack == 0 &&
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing == 0 &&
@@ -404,6 +427,13 @@ void p9_pgpe_ipc_sgpe_updt_active_quads(ipc_msg_t* cmd, void* arg)
             (ipcmsg_s2p_update_active_quads_t*)async_cmd->cmd_data;
         args->fields.return_code = SGPE_PGPE_RC_REQ_WHILE_PENDING_ACK;
         ipc_send_rsp(cmd, IPC_RC_SUCCESS);
+        PK_TRACE_INF("SGPE_UPDT_QUADS: %d %d\n",
+                     G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing,
+                     G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack);
+        PK_TRACE_INF("SGPE_UPDT_CORES: %d %d\n",
+                     G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing,
+                     G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack);
+        PK_TRACE_INF("SGPE_UPDT_QUADS: Acked\n");
     }
 
     if (G_pgpe_pstate_record.alreadySemPosted == 0)
@@ -412,5 +442,5 @@ void p9_pgpe_ipc_sgpe_updt_active_quads(ipc_msg_t* cmd, void* arg)
         G_pgpe_pstate_record.alreadySemPosted  = 1;
     }
 
-    PK_TRACE_DBG("SGPE_UPDT_QUADS: Exit\n");
+    PK_TRACE_INF("SGPE_UPDT_QUADS: Exit\n");
 }
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.c
index e371d552..90bf2bdd 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.c
@@ -44,7 +44,8 @@ const uint64_t ext_irq_vectors_gpe[NUM_EXT_IRQ_PRTY_LEVELS][2] =
 {
 
     {
-        IRQ_VEC_PRTY0_GPE,  IRQ_VEC_PRTY5_GPE2 |    /* 0 */
+        IRQ_VEC_PRTY0_GPE,  IRQ_VEC_PRTY6_GPE2 |    /* 0 */
+        IRQ_VEC_PRTY5_GPE2 |
         IRQ_VEC_PRTY4_GPE2 |
         IRQ_VEC_PRTY3_GPE2 |
         IRQ_VEC_PRTY2_GPE2 |
@@ -53,7 +54,8 @@ const uint64_t ext_irq_vectors_gpe[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     } ,
 
     {
-        IRQ_VEC_PRTY1_GPE2, IRQ_VEC_PRTY5_GPE2 |    /* 1 */
+        IRQ_VEC_PRTY1_GPE2, IRQ_VEC_PRTY6_GPE2 |    /* 1 */
+        IRQ_VEC_PRTY5_GPE2 |
         IRQ_VEC_PRTY4_GPE2 |
         IRQ_VEC_PRTY3_GPE2 |
         IRQ_VEC_PRTY2_GPE2 |
@@ -61,25 +63,32 @@ const uint64_t ext_irq_vectors_gpe[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     } ,
 
     {
-        IRQ_VEC_PRTY2_GPE2, IRQ_VEC_PRTY5_GPE2 |    /* 2 */
+        IRQ_VEC_PRTY2_GPE2, IRQ_VEC_PRTY6_GPE2 |    /* 2 */
+        IRQ_VEC_PRTY5_GPE2 |
         IRQ_VEC_PRTY4_GPE2 |
         IRQ_VEC_PRTY3_GPE2 |
         IRQ_VEC_PRTY2_GPE2
     } ,
 
     {
-        IRQ_VEC_PRTY3_GPE2, IRQ_VEC_PRTY5_GPE2 |    /* 2 */
+        IRQ_VEC_PRTY3_GPE2, IRQ_VEC_PRTY6_GPE2 |    /* 3 */
+        IRQ_VEC_PRTY5_GPE2 |
         IRQ_VEC_PRTY4_GPE2 |
         IRQ_VEC_PRTY3_GPE2
     } ,
 
     {
-        IRQ_VEC_PRTY4_GPE2, IRQ_VEC_PRTY5_GPE2 |    /* 2 */
+        IRQ_VEC_PRTY4_GPE2, IRQ_VEC_PRTY6_GPE2 |    /* 4 */
+        IRQ_VEC_PRTY5_GPE2 |
         IRQ_VEC_PRTY4_GPE2
     } ,
-    { IRQ_VEC_PRTY5_GPE2, IRQ_VEC_PRTY5_GPE2 } ,  /* 3 */
 
-    { IRQ_VEC_PRTY6_GPE2, IRQ_VEC_PRTY6_GPE2 }    /* 4 */
+    {
+        IRQ_VEC_PRTY5_GPE2, IRQ_VEC_PRTY6_GPE2 |    /* 5 */
+        IRQ_VEC_PRTY5_GPE2
+    } ,
+
+    { IRQ_VEC_PRTY6_GPE2, IRQ_VEC_PRTY6_GPE2 }      /* 6 */
 
 };
 
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.h
index 22fb6f89..0fecda0b 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq.h
@@ -77,11 +77,12 @@ extern const uint64_t ext_irq_vectors_gpe[NUM_EXT_IRQ_PRTY_LEVELS][2];
 #define IRQ_VEC_PRTY2_GPE2  (uint64_t)(0x0001000000000000) // Task2-CHECK_STOP_GPE2
 #define IRQ_VEC_PRTY3_GPE2  (uint64_t)(0x0000000000000008) // Task3-IPI2-LO(Process Flags)
 #define IRQ_VEC_PRTY4_GPE2  (uint64_t)(0x0000001000000000) // Task4-IPI2-HI(IPC from OCC/SGPE)
-#define IRQ_VEC_PRTY5_GPE2  (uint64_t)(0x0000000000020000) // Task5-PCB_INTR_TYPE1(PCB Type1 from CME)
+#define IRQ_VEC_PRTY5_GPE2  (uint64_t)(0x0000000000004000) // Task5-PCB_INTR_TYPE4(PCB Type4 from CME)
+#define IRQ_VEC_PRTY6_GPE2  (uint64_t)(0x0000000000020000) // Task5-PCB_INTR_TYPE1(PCB Type1 from CME)
 #if OVERRIDE_OTHER_ENGINES_IRQS == 1
-    #define IRQ_VEC_PRTY6_GPE2  (uint64_t)(0xDF7EFF03FFFDFFF5) // Other instances' IRQs
+    #define IRQ_VEC_PRTY7_GPE2  (uint64_t)(0xDF7EFF03BFFDFFF5) // Other instances' IRQs
 #else
-    #define IRQ_VEC_PRTY6_GPE2  (uint64_t)(0x0000000000000000) // Other instances' IRQs
+    #define IRQ_VEC_PRTY7_GPE2  (uint64_t)(0x0000000000000000) // Other instances' IRQs
 #endif
 // Unique to each instance
 // We should never detect these
@@ -91,10 +92,11 @@ extern const uint64_t ext_irq_vectors_gpe[NUM_EXT_IRQ_PRTY_LEVELS][2];
                                 IRQ_VEC_PRTY2_GPE2 | \
                                 IRQ_VEC_PRTY3_GPE2 | \
                                 IRQ_VEC_PRTY4_GPE2 | \
-                                IRQ_VEC_PRTY5_GPE2 )      // Note, we do not incl PRTY6 here!
+                                IRQ_VEC_PRTY5_GPE2 | \
+                                IRQ_VEC_PRTY6_GPE2 )      // Note, we do not incl PRTY7 here!
 
 #define IRQ_VEC_PRTY_CHECK    ( IRQ_VEC_ALL_OUR_IRQS | \
-                                IRQ_VEC_PRTY6_GPE2 )      // This should be 0xFFFFFFFFFFFFFFFF
+                                IRQ_VEC_PRTY7_GPE2 )      // This should be 0xFFFFFFFFFFFFFFFF
 
 extern uint8_t    g_current_prty_level;
 extern uint8_t    g_oimr_stack[NUM_EXT_IRQ_PRTY_LEVELS];
@@ -104,6 +106,8 @@ extern uint64_t   g_oimr_override;
 
 void pk_irq_save_and_set_mask(uint32_t iPrtyLvl);
 
+
+
 /// Restore a vector of interrupts by overwriting OIMR.
 UNLESS__PPE42_IRQ_CORE_C__(extern)
 inline void
@@ -137,4 +141,26 @@ pk_irq_vec_restore( PkMachineContext* context)
     pk_critical_section_exit(context);
 }
 
+//As per PPE SPEC, the Fixed-Interval Timer and Decrementer Interrupt
+//are lower priority than external interrupts.
+//In some external interrupts handlers atomiticy might be required while
+//keeping FIT and DEC unmasked. The sub-critical section concepts can be
+//helpful, and the following the two functions can be used for such purpose.
+//
+//PGPE makes use of this in several places
+UNLESS__PPE42_IRQ_CORE_C__(extern)
+inline void pk_irq_sub_critical_enter(PkMachineContext* ctx)
+{
+
+    pk_critical_section_enter(ctx);
+    pk_irq_save_and_set_mask(0);
+    pk_critical_section_exit(ctx);
+}
+
+UNLESS__PPE42_IRQ_CORE_C__(extern)
+inline void pk_irq_sub_critical_exit(PkMachineContext* ctx)
+{
+
+    pk_irq_vec_restore(ctx);
+}
 #endif // _P9_PGPE_IRQ_H_
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
index 9072d28b..d05f737f 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
@@ -26,11 +26,15 @@
 #include "p9_pgpe.h"
 #include "p9_pgpe_pstate.h"
 #include "pstate_pgpe_occ_api.h"
+#include "p9_dd1_doorbell_wr.h"
+#include "ppe42_cache.h"
+#include "p9_pgpe_gppb.h"
 
 //
 //External Global Data
 //
 extern PgpePstateRecord G_pgpe_pstate_record;
+extern GlobalPstateParmBlock* G_gppb;
 
 //
 //OCB Error Interrupt Handler
@@ -122,8 +126,6 @@ void p9_pgpe_irq_handler_xstop_gpe2(void* arg, PkIrqId irq)
 //
 //IPI2 Lo Priority Interrupt Handler
 //
-//\TODO: RTC 164107
-//Implement this handler. Should call "process_flags" function(needs to be implemented).
 void p9_pgpe_irq_handler_ipi2_lo(void* arg, PkIrqId irq)
 {
     PK_TRACE_DBG("IPI2 Lo: Enter\n");
@@ -210,3 +212,124 @@ void p9_pgpe_irq_handler_pcb_type1(void* arg, PkIrqId irq)
     pk_irq_vec_restore(&ctx);//Restore interrupts
     PK_TRACE_DBG("PCB_TYPE1: Exit\n");
 }
+
+//
+//PCB Type 4 Interrupt Handler
+//
+void p9_pgpe_irq_handler_pcb_type4(void* arg, PkIrqId irq)
+{
+    PkMachineContext ctx;
+    //We don't want the register and start doorbell to be interrupted by any external interrupt.
+    //However, we still want FIT Timer to be active, so we mask of all interrupts
+    //in the OIMR through UIH(by setting UIH priority lvl = 0)
+    pk_critical_section_enter(&ctx);
+    pk_irq_save_and_set_mask(0);
+    pk_critical_section_exit(&ctx);
+
+
+    PK_TRACE_DBG("PCB_TYPE4: Enter\n");
+    ocb_ccsr_t ccsr;
+    ccsr.value = in32(OCB_CCSR);
+    uint32_t quadAckExpect = 0;
+    volatile uint32_t opit4pr, opit4pr1;
+    uint32_t opit4prQuad, q, c;
+    pgpe_db0_start_ps_bcast_t db0;
+    db0.value = 0;
+    db0.fields.msg_id = MSGID_DB0_START_PSTATE_BROADCAST;
+
+    g_oimr_override |= BIT64(49);
+    out32(OCB_OIMR1_OR, BIT32(17)); //Disable PCB_INTR_TYPE4
+    opit4pr = in32(OCB_OPIT4PRA);
+    out32(OCB_OPIT4PRA_CLR, opit4pr);
+    PK_TRACE_DBG("PCB_TYPE4: opit4pr 0x%x\n", opit4pr);
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        opit4prQuad = (opit4pr >> ((MAX_QUADS - q + 1) << 2)) & 0xf;
+        PK_TRACE_DBG("PCB_TYPE4: opit4prQuad 0x%x\n", opit4prQuad);
+
+        if (opit4prQuad)
+        {
+            //Already registered
+            if (G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+            {
+                PK_TRACE_DBG("PCB_TYPE4: Already Registered\n");
+                pk_halt();
+            }
+
+            if (ccsr.value & ((0xF0000000) >> (q << 2)))
+            {
+                G_pgpe_pstate_record.quadsActive |= (0x80 >> q);
+                p9_pgpe_pstate_do_auction(ALL_QUADS_BIT_MASK);
+                p9_pgpe_pstate_apply_clips();
+
+                db0.fields.global_actual = G_pgpe_pstate_record.globalPSTarget;
+                db0.fields.quad0_ps = G_pgpe_pstate_record.quadPSTarget[0];
+                db0.fields.quad1_ps = G_pgpe_pstate_record.quadPSTarget[1];
+                db0.fields.quad2_ps = G_pgpe_pstate_record.quadPSTarget[2];
+                db0.fields.quad3_ps = G_pgpe_pstate_record.quadPSTarget[3];
+                db0.fields.quad4_ps = G_pgpe_pstate_record.quadPSTarget[4];
+                db0.fields.quad5_ps = G_pgpe_pstate_record.quadPSTarget[5];
+
+                for (c = q << 2; c < ((q + 1) << 2); c++)
+                {
+                    if (ccsr.value & ((0x80000000) >> c))
+                    {
+                        opit4pr1 = in32(OCB_OPIT4PRA);
+                        PK_TRACE_DBG("PCB_TYPE4: opit4pr 0x%x, quadAckExpect=0x%x\n", opit4pr1, quadAckExpect);
+                        p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB0, c), db0.value);
+                        PK_TRACE_DBG("PCB_TYPE4: Sent DB0 to %d\n", q);
+                    }
+                }
+            }
+
+            quadAckExpect |= (0x80 >> q);
+        }
+    }
+
+    opit4pr1 = in32(OCB_OPIT4PRA);
+    PK_TRACE_DBG("PCB_TYPE4: opit4pr 0x%x, quadAckExpect=0x%x\n", opit4pr1, quadAckExpect);
+
+
+    //Now, wait for all Pstate Start DB0 to be ACKed
+    while(quadAckExpect != 0)
+    {
+        opit4pr1 = in32(OCB_OPIT4PRA);
+
+
+        for (q = 0; q < MAX_QUADS; q++)
+        {
+            opit4prQuad = (opit4pr1 >> ((MAX_QUADS - q + 1) << 2)) & 0xf;
+
+            if (opit4prQuad)
+            {
+                PK_TRACE_DBG("PCB_TYPE4: opit4prQuad 0x%x\n", opit4prQuad);
+
+                if (quadAckExpect & (0x80 >> q))
+                {
+                    quadAckExpect &= ~(0x80 >> q);
+                    out32(OCB_OPIT4PRA_CLR, opit4prQuad << ((MAX_QUADS - q + 1) << 2));
+                    PK_TRACE_DBG("PCB_TYPE4: Got DB0 ACK from %d\n", q);
+                    G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                }
+                else
+                {
+                    pk_halt();
+                }
+            }
+        }
+    }
+
+    PK_TRACE_DBG("PCB_TYPE4: opit4pr 0x%x, quadAckExpect=0x%x\n", opit4pr1, quadAckExpect);
+    out32(OCB_OISR1_CLR, BIT32(17));
+    g_oimr_override &= ~BIT64(49);
+    out32(OCB_OIMR1_CLR, BIT32(17)); //Disable PCB_INTR_TYPE4
+
+    //Now, that type4 processing is complete, let's
+    //restore all the interrupts we masked at the top
+    pk_irq_vec_restore(&ctx);
+
+    pk_irq_vec_restore(&ctx);//Second call is to restore type4 interrupt itself
+    PK_TRACE_DBG("PCB_TYPE4: Exit\n");
+}
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
index 29a5c1db..f2aa4e02 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
@@ -52,6 +52,8 @@ PgpePstateRecord G_pgpe_pstate_record __attribute__((section (".dump_ptrs"))) =
     {0, 0, 0, 0, 0, 0},
     0,
     0,
+    {0, 0, 0, 0, 0, 0},
+    {0, 0},
     0,
     0,
     0,
@@ -116,7 +118,7 @@ IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE0_PENDING
 IRQ_HANDLER(p9_pgpe_irq_handler_pcb_type1, NULL) //OCCHW_IRQ_PMC_PCB_INTR_TYPE1_PENDING
 IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE2_PENDING
 IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE3_PENDING
-IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE4_PENDING
+IRQ_HANDLER(p9_pgpe_irq_handler_pcb_type4, NULL) //OCCHW_IRQ_PMC_PCB_INTR_TYPE4_PENDING
 IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE5_PENDING
 IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE6_PENDING
 IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_PMC_PCB_INTR_TYPE7_PENDING
@@ -233,8 +235,17 @@ main(int argc, char** argv)
     PK_TRACE_DBG("Setup OCCFIR[OCC_HB Error]");
     p9_pgpe_ocb_hb_error_init();
 
+    g_oimr_override |= BIT64(49);
+    out32(OCB_OIMR1_OR, BIT32(17)); //Disable PCB_INTR_TYPE4
+
     PK_TRACE_DBG("Start PK Threads");
 
+    uint64_t data;
+    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_CTRL, 3), data);
+    PK_TRACE_DBG("Got Data=0x%08x%08x", data >> 32, data);
+    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, 3), data);
+    PK_TRACE_DBG("Got Data=0x%08x%08x", data >> 32, data);
+    //GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, 3),data);
     // Start running the highest priority thread.
     // This function never returns
     pk_start_threads();
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
index 0aa4670b..e4841f98 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
@@ -59,7 +59,6 @@ GPE_BUFFER(ipcmsg_s2p_suspend_pstate_t G_sgpe_suspend_stop);
 //Local Functions
 inline void p9_pgpe_pstate_freq_updt(uint32_t activeCores);
 void p9_pgpe_suspend_stop_callback(ipc_msg_t* msg, void* arg);
-void p9_pgpe_wait_cme_db_ack(uint8_t msg_id, uint32_t activeCores);
 
 //
 //p9_pgpe_pstate_init
@@ -67,6 +66,8 @@ void p9_pgpe_wait_cme_db_ack(uint8_t msg_id, uint32_t activeCores);
 void p9_pgpe_pstate_init()
 {
     uint32_t q;
+    ocb_qcsr_t qcsr;
+    qcsr.value = in32(OCB_QCSR);
 
     G_pgpe_pstate_record.pstatesStatus = PSTATE_INIT;
     G_pgpe_pstate_record.wofEnabled = 0;
@@ -84,6 +85,12 @@ void p9_pgpe_pstate_init()
         G_pgpe_pstate_record.globalPSCurr  = G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate;
         G_pgpe_pstate_record.quadPSNext[q] = G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate;
         G_pgpe_pstate_record.globalPSNext  = G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate;
+
+        //Set quads active equal to configured quads
+        if (qcsr.fields.ex_config & (0xC00 >> (q << 1)))
+        {
+            G_pgpe_pstate_record.quadsActive |= (0x80 >> q);
+        }
     }
 
     G_pgpe_pstate_record.pQuadState0 = (quad_state0_t*)G_pgpe_header_data->g_quad_status_addr;
@@ -122,12 +129,12 @@ void p9_pgpe_pstate_update(uint8_t* s)
 //
 void p9_pgpe_pstate_do_auction(uint8_t quadAuctionRequest)
 {
-    PK_TRACE_INF("AUCT: Enter");
+    PK_TRACE_DBG("AUCT: Enter");
     //Get active cores and quads
     uint32_t q, c;
     uint32_t activeCores = G_pgpe_pstate_record.pQuadState0->fields.active_cores;
     activeCores = (activeCores << 16) | (G_pgpe_pstate_record.pQuadState1->fields.active_cores);
-    uint32_t activeQuads = G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads;
+    uint32_t activeQuads = G_pgpe_pstate_record.quadsActive;
 
     //Local PStates Auction
     for (q = 0; q < MAX_QUADS; q++)
@@ -156,7 +163,7 @@ void p9_pgpe_pstate_do_auction(uint8_t quadAuctionRequest)
             G_pgpe_pstate_record.quadPSComputed[q] = 0xFF;
         }
 
-        PK_TRACE_INF("AUCTION: G_quadPSComputed: 0x%x", G_pgpe_pstate_record.quadPSComputed[q]);
+        PK_TRACE_DBG("AUCTION: G_quadPSComputed: 0x%x", G_pgpe_pstate_record.quadPSComputed[q]);
     }
 
     //Global PState Auction
@@ -174,7 +181,7 @@ void p9_pgpe_pstate_do_auction(uint8_t quadAuctionRequest)
     }
 
     PK_TRACE_INF("AUCT glbPSCmpted: 0x%x", G_pgpe_pstate_record.globalPSComputed);
-    PK_TRACE_INF("AUCT: Exit");
+    PK_TRACE_DBG("AUCT: Exit");
 }
 
 //
@@ -182,18 +189,17 @@ void p9_pgpe_pstate_do_auction(uint8_t quadAuctionRequest)
 //
 void p9_pgpe_pstate_apply_clips()
 {
-    PK_TRACE_INF("APCLP: Enter");
+    PK_TRACE_DBG("APCLP: Enter");
     uint32_t q;
-    uint32_t activeQuads = G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads;
+    uint32_t activeQuads = G_pgpe_pstate_record.quadsActive;
 
     //Apply clips to quad computed
-    PK_TRACE_INF("APCLP: 0x%x", activeQuads);
+    PK_TRACE_DBG("APCLP: 0x%x", activeQuads);
 
     for (q = 0; q < MAX_QUADS; q++)
     {
         uint8_t minPS = G_pgpe_pstate_record.psClipMin[q];
 
-//        G_pgpe_pstate_record.quadPSTarget[q] = G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate;
 
         if (activeQuads & (QUAD0_BIT_MASK >> q))
         {
@@ -217,11 +223,14 @@ void p9_pgpe_pstate_apply_clips()
             else
             {
                 G_pgpe_pstate_record.quadPSTarget[q] = G_pgpe_pstate_record.quadPSComputed[q];
+                PK_TRACE_INF("APCLP: neither");
             }
         }
         else
         {
             G_pgpe_pstate_record.quadPSTarget[q] = 0xFF;
+            G_pgpe_pstate_record.quadPSCurr[q] = 0xFF;
+            G_pgpe_pstate_record.quadPSNext[q] = 0xFF;
         }
 
         PK_TRACE_INF("APCLP: qPSTgt: 0x%x,cl=0x%x,0x%x", G_pgpe_pstate_record.quadPSTarget[q], minPS,
@@ -243,7 +252,7 @@ void p9_pgpe_pstate_apply_clips()
     }
 
     PK_TRACE_INF("APCLP: glbPSTgt: 0x%x", G_pgpe_pstate_record.globalPSTarget);
-    PK_TRACE_INF("APCLP: Exit");
+    PK_TRACE_DBG("APCLP: Exit");
 }
 
 //
@@ -251,8 +260,6 @@ void p9_pgpe_pstate_apply_clips()
 //
 void p9_pgpe_pstate_calc_wof()
 {
-    //\TODO RTC 162896
-    //Figure out how to calculate WOF Clip
     G_pgpe_pstate_record.wofClip = G_pgpe_pstate_record.pVFRT->vfrt_data[0][23];
 
     p9_pgpe_pstate_apply_clips();
@@ -268,6 +275,120 @@ void p9_pgpe_pstate_update_wof_state()
     wof_state->fields.vratio = 1;
 }
 
+//
+void p9_pgpe_pstate_process_quad_entry(uint32_t quadsAffected)
+{
+    uint32_t q;
+
+    G_pgpe_pstate_record.quadsActive &= ~quadsAffected;
+
+    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_ACTIVE)
+    {
+        for (q = 0; q < MAX_QUADS; q++)
+        {
+            if (quadsAffected & (QUAD0_BIT_MASK >> q))
+            {
+                G_pgpe_pstate_record.quadPSTarget[q] = 0xFF;
+                G_pgpe_pstate_record.quadPSCurr[q] = 0xFF;
+                G_pgpe_pstate_record.quadPSNext[q] = 0xFF;
+                G_pgpe_pstate_record.quadPSComputed[q] = 0xFF;
+            }
+        }
+
+        G_pgpe_pstate_record.pQuadState0->fields.quad0_pstate = G_pgpe_pstate_record.quadPSCurr[0];
+        G_pgpe_pstate_record.pQuadState0->fields.quad1_pstate = G_pgpe_pstate_record.quadPSCurr[1];
+        G_pgpe_pstate_record.pQuadState0->fields.quad2_pstate = G_pgpe_pstate_record.quadPSCurr[2];
+        G_pgpe_pstate_record.pQuadState0->fields.quad3_pstate = G_pgpe_pstate_record.quadPSCurr[3];
+        G_pgpe_pstate_record.pQuadState1->fields.quad4_pstate = G_pgpe_pstate_record.quadPSCurr[4];
+        G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = G_pgpe_pstate_record.quadPSCurr[5];
+
+        p9_pgpe_pstate_do_auction(ALL_QUADS_BIT_MASK);
+        p9_pgpe_pstate_apply_clips();
+
+        //Stop Pending
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack == 1)
+        {
+
+            ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+            ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+
+            //If Start
+            if (args->action == PGPE_ACTION_PSTATE_START)
+            {
+                p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
+            }
+            else
+            {
+                p9_pgpe_pstate_stop();
+            }
+        }
+        else
+        {
+            G_pgpe_pstate_record.pstatesStatus = PSTATE_ACTIVE;
+        }
+
+        pk_semaphore_post(&G_pgpe_pstate_record.sem_actuate);
+    }
+    else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_STOPPED_INIT)
+    {
+        //Start Pending
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack == 1)
+        {
+            p9_pgpe_pstate_start(PSTATE_START_OCC_IPC);
+            pk_semaphore_post(&G_pgpe_pstate_record.sem_actuate);
+        }
+        else
+        {
+            G_pgpe_pstate_record.pstatesStatus = PSTATE_STOPPED;
+        }
+
+    }
+
+    PK_TRACE_DBG("PS: Q Updt Entry Done.q=%x\n", G_pgpe_pstate_record.quadsActive);
+}
+
+void p9_pgpe_pstate_process_quad_exit(uint32_t quadsAffected)
+{
+    uint32_t q;
+    qppm_dpll_freq_t dpllFreq;
+    dpllFreq.value = 0;
+    qppm_qpmmr_t qpmmr;
+    uint32_t cme_interppm_dpll_enable_set = 0;
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if (quadsAffected & (QUAD0_BIT_MASK >> q))
+        {
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR, q), qpmmr.value);
+            PK_TRACE_DBG("PS: Q Updt Exit qpmmr.0x%08x\n", qpmmr.value >> 32);
+            cme_interppm_dpll_enable_set = qpmmr.fields.cme_interppm_dpll_enable;
+            qpmmr.fields.cme_interppm_dpll_enable = 0;
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR, q), qpmmr.value);
+
+            //Write QPPM_DPLL_FREQ for just woken up quads
+            dpllFreq.fields.fmax  = (uint16_t)(G_gppb->dpll_pstate0_value -
+                                               G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate);
+            dpllFreq.fields.fmult = (uint16_t)(G_gppb->dpll_pstate0_value -
+                                               G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate);
+            dpllFreq.fields.fmin  = (uint16_t)(G_gppb->dpll_pstate0_value -
+                                               G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][POWERSAVE].pstate);
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, q), dpllFreq.value);
+
+            qpmmr.fields.cme_interppm_dpll_enable = cme_interppm_dpll_enable_set;
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR, q), qpmmr.value);
+        }
+    }
+
+    //ACK back to SGPE
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd;
+    ipcmsg_s2p_update_active_quads_t* args = (ipcmsg_s2p_update_active_quads_t*)async_cmd->cmd_data;
+    args->fields.return_code = SGPE_PGPE_IPC_RC_SUCCESS;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack = 0;
+    ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd, IPC_RC_SUCCESS);
+
+    PK_TRACE_DBG("PS: Q Updt Exit Done.q=%x\n", G_pgpe_pstate_record.quadsActive);
+}
+
 void p9_pgpe_pstate_ipc_rsp_cb_sem_post(ipc_msg_t* msg, void* arg)
 {
     pk_semaphore_post((PkSemaphore*)arg);
@@ -278,7 +399,7 @@ void p9_pgpe_pstate_ipc_rsp_cb_sem_post(ipc_msg_t* msg, void* arg)
 //
 void p9_pgpe_pstate_pm_complex_suspend()
 {
-    PK_TRACE_INF("PS:PM Susp Enter");
+    PK_TRACE_DBG("PS:PM Susp Enter");
 
     if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
     {
@@ -297,7 +418,7 @@ void p9_pgpe_pstate_pm_complex_suspend()
         p9_pgpe_pstate_send_suspend_stop();
     }
 
-    PK_TRACE_INF("PS:PM Susp Exit");
+    PK_TRACE_DBG("PS:PM Susp Exit");
 }
 
 //
@@ -348,7 +469,7 @@ void p9_pgpe_suspend_stop_callback(ipc_msg_t* msg, void* arg)
 //
 void p9_pgpe_pstate_safe_mode()
 {
-    PK_TRACE_INF("PS:Safe Mode Enter");
+    PK_TRACE_DBG("PS:Safe Mode Enter");
 
     if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
     {
@@ -366,7 +487,7 @@ void p9_pgpe_pstate_safe_mode()
         G_pgpe_pstate_record.pstatesStatus = PSTATE_SAFE_MODE;
     }
 
-    PK_TRACE_INF("PS:Safe Mode Exit");
+    PK_TRACE_DBG("PS:Safe Mode Exit");
 }
 
 //
@@ -374,7 +495,7 @@ void p9_pgpe_pstate_safe_mode()
 //
 void p9_pgpe_pstate_apply_safe_clips()
 {
-    PK_TRACE_INF("PS: Apply Safe Enter");
+    PK_TRACE_DBG("PS: Apply Safe Enter");
     int q;
 
     //Set clips to safe pstate
@@ -387,7 +508,7 @@ void p9_pgpe_pstate_apply_safe_clips()
     //Update clips
     p9_pgpe_pstate_apply_clips();
 
-    PK_TRACE_INF("PS: Apply Safe Exit");
+    PK_TRACE_DBG("PS: Apply Safe Exit");
 }
 
 //
@@ -396,12 +517,18 @@ void p9_pgpe_pstate_apply_safe_clips()
 int32_t p9_pgpe_pstate_at_target()
 {
     if( G_pgpe_pstate_record.globalPSCurr != G_pgpe_pstate_record.globalPSTarget ||
-        G_pgpe_pstate_record.quadPSCurr[0] ^ G_pgpe_pstate_record.quadPSTarget[0] ||
-        G_pgpe_pstate_record.quadPSCurr[1] ^ G_pgpe_pstate_record.quadPSTarget[1] ||
-        G_pgpe_pstate_record.quadPSCurr[2] ^ G_pgpe_pstate_record.quadPSTarget[2] ||
-        G_pgpe_pstate_record.quadPSCurr[3] ^ G_pgpe_pstate_record.quadPSTarget[3] ||
-        G_pgpe_pstate_record.quadPSCurr[4] ^ G_pgpe_pstate_record.quadPSTarget[4] ||
-        G_pgpe_pstate_record.quadPSCurr[5] ^ G_pgpe_pstate_record.quadPSTarget[5] )
+        ((G_pgpe_pstate_record.quadsActive & QUAD0_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[0] ^ G_pgpe_pstate_record.quadPSTarget[0]) ||
+        ((G_pgpe_pstate_record.quadsActive & QUAD1_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[1] ^ G_pgpe_pstate_record.quadPSTarget[1]) ||
+        ((G_pgpe_pstate_record.quadsActive & QUAD2_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[2] ^ G_pgpe_pstate_record.quadPSTarget[2]) ||
+        ((G_pgpe_pstate_record.quadsActive & QUAD3_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[3] ^ G_pgpe_pstate_record.quadPSTarget[3]) ||
+        ((G_pgpe_pstate_record.quadsActive & QUAD4_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[4] ^ G_pgpe_pstate_record.quadPSTarget[4]) ||
+        ((G_pgpe_pstate_record.quadsActive & QUAD5_BIT_MASK)
+         && G_pgpe_pstate_record.quadPSCurr[5] ^ G_pgpe_pstate_record.quadPSTarget[5]) )
     {
         return 0;
     }
@@ -416,32 +543,21 @@ int32_t p9_pgpe_pstate_at_target()
 //
 void p9_pgpe_pstate_do_step()
 {
-    PkMachineContext ctx;
-    //We don't want actuate_step to interrupted by any external interrupt.
-    //However, we still want FIT Timer to be active, so we mask of all interrupts
-    //in the OIMR through UIH(by setting UIH priority lvl = 0)
-    pk_critical_section_enter(&ctx);
-    pk_irq_save_and_set_mask(0);
-    pk_critical_section_exit(&ctx);
-
     //Do one actuate step
-    PK_TRACE_DBG("ACT_TH: PSTgt: 0x%x PSCurr: 0x%x", G_pgpe_pstate_record.globalPSTarget,
-                 G_pgpe_pstate_record.globalPSCurr);
-
-    PK_TRACE_DBG("ACT_TH: Step Entry");
-    PK_TRACE_DBG("ACT_TH: PSTgt: 0x%x PSCurr: 0x%x", G_pgpe_pstate_record.globalPSTarget,
+    PK_TRACE_DBG("PS: Step Entry");
+    PK_TRACE_DBG("PS: PSTgt: 0x%x PSCurr: 0x%x", G_pgpe_pstate_record.globalPSTarget,
                  G_pgpe_pstate_record.globalPSCurr);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[0],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[0],
                  G_pgpe_pstate_record.quadPSCurr[0]);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[1],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[1],
                  G_pgpe_pstate_record.quadPSCurr[1]);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[2],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[2],
                  G_pgpe_pstate_record.quadPSCurr[2]);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[3],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[3],
                  G_pgpe_pstate_record.quadPSCurr[3]);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[4],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[4],
                  G_pgpe_pstate_record.quadPSCurr[4]);
-    PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[5],
+    PK_TRACE_DBG("PS: QTgt: 0x%x QCurr: 0x%x", G_pgpe_pstate_record.quadPSTarget[5],
                  G_pgpe_pstate_record.quadPSCurr[5]);
     uint32_t q;
     ocb_qcsr_t qcsr;
@@ -453,7 +569,7 @@ void p9_pgpe_pstate_do_step()
     //Determine active and configured cores
     for (q = 0; q < MAX_QUADS; q++)
     {
-        if(G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q))
+        if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
         {
             if (qcsr.fields.ex_config & (0x800 >> (q << 1)))
             {
@@ -466,7 +582,7 @@ void p9_pgpe_pstate_do_step()
             }
         }
 
-        PK_TRACE_INF("ACT_TH: act_c=0x%x", active_conf_cores);
+        PK_TRACE_INF("PS: act_c=0x%x", active_conf_cores);
     }
 
     //Higher number PState
@@ -479,7 +595,10 @@ void p9_pgpe_pstate_do_step()
 
             for (q = 0; q < MAX_QUADS; q++)
             {
-                G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+                {
+                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                }
             }
         }
         else
@@ -489,13 +608,16 @@ void p9_pgpe_pstate_do_step()
 
             for (q = 0; q < MAX_QUADS; q++)
             {
-                if (G_pgpe_pstate_record.quadPSTarget[q] > G_pgpe_pstate_record.globalPSNext)   //Keep localPS under GlobalPS
+                if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
                 {
-                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.globalPSNext;
-                }
-                else
-                {
-                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                    if (G_pgpe_pstate_record.quadPSTarget[q] > G_pgpe_pstate_record.globalPSNext)   //Keep localPS under GlobalPS
+                    {
+                        G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.globalPSNext;
+                    }
+                    else
+                    {
+                        G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                    }
                 }
             }
         }
@@ -513,7 +635,10 @@ void p9_pgpe_pstate_do_step()
 
             for (q = 0; q < MAX_QUADS; q++)
             {
-                G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+                {
+                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                }
             }
         }
         else
@@ -523,13 +648,16 @@ void p9_pgpe_pstate_do_step()
 
             for (q = 0; q < MAX_QUADS; q++)
             {
-                if (G_pgpe_pstate_record.quadPSTarget[q] < G_pgpe_pstate_record.globalPSNext)   //Keep localPS under GlobalPS
-                {
-                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.globalPSNext;
-                }
-                else
+                if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
                 {
-                    G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                    if (G_pgpe_pstate_record.quadPSTarget[q] < G_pgpe_pstate_record.globalPSNext)   //Keep localPS under GlobalPS
+                    {
+                        G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.globalPSNext;
+                    }
+                    else
+                    {
+                        G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                    }
                 }
             }
         }
@@ -541,7 +669,10 @@ void p9_pgpe_pstate_do_step()
     {
         for (q = 0; q < MAX_QUADS; q++)
         {
-            G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+            if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+            {
+                G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+            }
         }
 
         p9_pgpe_pstate_freq_updt(active_conf_cores);
@@ -552,8 +683,7 @@ void p9_pgpe_pstate_do_step()
 
     for (q = 0; q < MAX_QUADS; q++)
     {
-        if(G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q) &&
-           (qcsr.fields.ex_config & (0xC00 >> 2 * q)))
+        if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
         {
             G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSNext[q];
         }
@@ -567,20 +697,17 @@ void p9_pgpe_pstate_do_step()
     G_pgpe_pstate_record.pQuadState1->fields.quad4_pstate = G_pgpe_pstate_record.quadPSCurr[4];
     G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = G_pgpe_pstate_record.quadPSCurr[5];
 
-    /*    PK_TRACE_DBG("ACT_TH: PSTgt: 0x%x PSCurr: 0x%x", globalPSTarget, globalPSCurr);
-        PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", quadPSTarget[0], quadPSCurr[0]);
-        PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", quadPSTarget[1], quadPSCurr[1]);
-        PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", quadPSTarget[2], quadPSCurr[2]);
-        PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", quadPSTarget[3], quadPSCurr[3]);
-        PK_TRACE_DBG("ACT_TH: QTgt: 0x%x QCurr: 0x%x", quadPSTarget[4], quadPSCurr[4]);*/
-    PK_TRACE_DBG("ACT_TH: Step Exit");
-
+    PK_TRACE_DBG("PS: Step Exit");
 
-    //Now, PGPE is done with actuate step, let's
-    //restore the interrupts in OIMR through UIH
-    pk_irq_vec_restore(&ctx);
 }
 
+
+/*void p9_pgpe_pstate_do_suspend()
+{
+    //Ack back to SGPE and then sit in PSTATE_SUSPENDED loop until told otherwise
+    while (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED) {}
+}*/
+
 //
 //p9_pgpe_pstate__updt_ext_volt
 //
@@ -632,8 +759,11 @@ void p9_pgpe_pstate_updt_ext_volt(uint32_t tgtEVid)
 //Sends a DB0 to all active CMEs, so that Quad Managers(CMEs) update DPLL
 inline void p9_pgpe_pstate_freq_updt(uint32_t activeCores)
 {
-    PK_TRACE_INF("ACT_TH: FreqUpdt Enter");
-    PK_TRACE_INF("ACT_TH: act_cores=0x%x", activeCores);
+    PK_TRACE_DBG("PS: FreqUpdt Enter");
+    uint32_t q, c;
+    ocb_ccsr_t ccsr;
+    ccsr.value = in32(OCB_CCSR);
+
     pgpe_db0_glb_bcast_t db0;
     db0.value = 0;
     db0.fields.msg_id = MSGID_DB0_GLOBAL_ACTUAL_BROADCAST;
@@ -644,75 +774,60 @@ inline void p9_pgpe_pstate_freq_updt(uint32_t activeCores)
     db0.fields.quad3_ps = G_pgpe_pstate_record.quadPSNext[3];
     db0.fields.quad4_ps = G_pgpe_pstate_record.quadPSNext[4];
     db0.fields.quad5_ps = G_pgpe_pstate_record.quadPSNext[5];
-    p9_dd1_db_multicast_wr(PCB_MUTLICAST_GRP1 | CPPM_CMEDB0, db0.value, activeCores);
-    PK_TRACE_INF("ACT_TH: MCAST DB0");
-    p9_pgpe_wait_cme_db_ack(MSGID_DB0_GLOBAL_ACTUAL_BROADCAST, activeCores);//Wait for ACKs from all CMEs
-    PK_TRACE_INF("ACT_TH: Freq Updt Exit");
-}
-
-
-//
-//Wait for CME to ack DB0
-//
-void p9_pgpe_wait_cme_db_ack(uint8_t msg_id, uint32_t active_conf_cores)
-{
-    int32_t c, q;
-    uint32_t opit4pr;
-    uint8_t pendingAcks = 0;
-    uint8_t quadPendDBAck[MAX_QUADS];
 
+    //p9_dd1_db_multicast_wr(PCB_MUTLICAST_GRP1 | CPPM_CMEDB0, db0.value, activeCores);
     for (q = 0; q < MAX_QUADS; q++)
     {
-        if (active_conf_cores & (0xF0000000 >> (q << 2)))
-        {
-            quadPendDBAck[q] = 1;
-        }
-        else
+        if (G_pgpe_pstate_record.quadsActive & (0x80 >> q))
         {
-            quadPendDBAck[q] = 0;
+            for (c = q << 2; c < ((q + 1) << 2); c++)
+            {
+                if (ccsr.value & ((0xc0000000) >> c))
+                {
+                    p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB0, c), db0.value);
+                }
+            }
         }
     }
 
-    //Wait for all the acks to come
-    pendingAcks = 1;
+    p9_pgpe_wait_cme_db_ack(G_pgpe_pstate_record.quadsActive);//Wait for ACKs from all QuadManagers
+    PK_TRACE_DBG("PS: Freq Updt Exit");
+}
+
+void p9_pgpe_wait_cme_db_ack(uint32_t quadAckExpect)
+{
+    uint32_t q;
+    uint32_t opit4pr, opit4prQuad, opit4Clr = 0;
+
+    PK_TRACE_INF("PS: AckExpect=0x%x", quadAckExpect);
 
-    while (pendingAcks)
+    //Now, wait for all Pstate Start DB0 to be ACKed
+    while(quadAckExpect != 0)
     {
-        //Process acks
         opit4pr = in32(OCB_OPIT4PRA);
 
-        for (c = 0; c < MAX_CORES; c++)
+        for (q = 0; q < MAX_QUADS; q++)
         {
-            if (opit4pr & (CCSR_CORE_CONFIG_MASK >> c))
+            if (quadAckExpect & (0x80 >> q))
             {
-                if (quadPendDBAck[QUAD_FROM_CORE(c)])
-                {
-                    quadPendDBAck[QUAD_FROM_CORE(c)] = 0;
-                    out32(OCB_OPIT4PRA_CLR, CCSR_CORE_CONFIG_MASK >> c); //Clear out pending bits
-                    PK_TRACE_INF("ACT_TH: Ack C%02u", c);
-                }
-                else
+                opit4prQuad = (opit4pr >> ((MAX_QUADS - q + 1) << 2)) & 0xf;
+
+                if (opit4prQuad)
                 {
-                    PK_TRACE_INF("ACT_TH: BadAck C%02u", c);
-                    pk_halt();
+                    PK_TRACE_INF("PS: opit4prQuad=0x%x", opit4pr);
+                    quadAckExpect &= ~(0x80 >> q);
+                    opit4Clr |= (opit4prQuad << ((MAX_QUADS - q + 1) << 2));
+                    PK_TRACE_INF("PS: GotAck from %d", q);
                 }
             }
         }
+    }
 
-        //Check if any pending left
-        pendingAcks = 0;
+    out32(OCB_OPIT4PRA_CLR, opit4Clr);
 
-        for (q = 0; q < MAX_QUADS; q++)
-        {
-            if (quadPendDBAck[q])
-            {
-                pendingAcks = 1;
-            }
-        }
-    }
+    PK_TRACE_INF("PS: qCME ACKs rcvd");
 }
 
-
 void p9_pgpe_pstate_set_pmcr_owner(uint32_t owner)
 {
     int q = 0;
@@ -757,3 +872,292 @@ void p9_pgpe_pstate_set_pmcr_owner(uint32_t owner)
 
 #endif
 }
+
+//
+//p9_pgpe_pstate_start()
+//
+void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
+{
+    PK_TRACE_DBG("PS: Start Enter");
+    ocb_ccsr_t ccsr;
+    ccsr.value = in32(OCB_CCSR);
+    uint8_t quadPstates[MAX_QUADS];
+    uint32_t lowestDpll, syncPstate;
+    qppm_dpll_stat_t dpll;
+    uint32_t active_conf_cores = 0;
+    uint32_t q, c;
+
+    //Setup Shared Memory Area
+    G_pgpe_pstate_record.pQuadState0->fields.active_cores = (ccsr.value >> 16);
+    G_pgpe_pstate_record.pQuadState1->fields.active_cores = (ccsr.value & 0xFF00);
+    G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads =  G_pgpe_pstate_record.quadsActive;
+    PK_TRACE_DBG("PS: Shr Mem Updt");
+
+    //1. Read DPLLs
+    lowestDpll = 0xFFF;
+    dpll.value = 0;
+    PK_TRACE_INF("PS: DPLL0Value=0x%x", G_gppb->dpll_pstate0_value);
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+        {
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_STAT, q), dpll.value);
+
+            if ((dpll.fields.freqout)  < lowestDpll )
+            {
+                lowestDpll = dpll.fields.freqout;
+            }
+
+            PK_TRACE_DBG("PS: Quad[%d]: DPLL=0x%x", q, (dpll.fields.freqout));
+            active_conf_cores |= (0xF0000000 >> q * 4);
+        }
+    }
+
+    //2. Determine Sync Pstate
+    if (lowestDpll > G_gppb->dpll_pstate0_value)
+    {
+        syncPstate = G_gppb->dpll_pstate0_value;
+    }
+    else
+    {
+        syncPstate = G_gppb->dpll_pstate0_value - lowestDpll;
+    }
+
+    //Init Core PStates
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+        {
+            quadPstates[q] = syncPstate;
+        }
+        else
+        {
+            quadPstates[q] = 0xFF;
+        }
+    }
+
+    p9_pgpe_pstate_update(quadPstates);
+    PK_TRACE_INF("PS: Sync Pstate=0x%x", syncPstate);
+    PK_TRACE_INF("PS: LowDPLL:0x%x", lowestDpll);
+    PK_TRACE_INF("PS: GlblPsTgt:0x%x", G_pgpe_pstate_record.globalPSTarget);
+
+    //3. Determine PMCR Owner
+    //We save it off so that CMEs that are currently in STOP11
+    //can be told upon STOP11 Exit
+    if (pstate_start_origin == PSTATE_START_OCC_IPC)
+    {
+        PK_TRACE_DBG("PS: g_oimr_override:0x%llx", g_oimr_override);
+        ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+        ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+
+        if (args->pmcr_owner == PMCR_OWNER_HOST)
+        {
+            PK_TRACE_DBG("PS: OWNER_HOST");
+            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_HOST;
+            g_oimr_override &= ~(BIT64(46));
+            out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
+        }
+        else if (args->pmcr_owner == PMCR_OWNER_OCC)
+        {
+            PK_TRACE_DBG("PS: OWNER_OCC");
+            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_OCC;
+            g_oimr_override |= BIT64(46);
+            out32(OCB_OIMR1_OR, BIT32(14)); //Disable PCB_INTR_TYPE1
+        }
+        else if (args->pmcr_owner == PMCR_OWNER_CHAR)
+        {
+            PK_TRACE_DBG("PS: OWNER_CHAR");
+            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_CHAR;
+            g_oimr_override &= ~(BIT64(46));
+            out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
+        }
+        else
+        {
+            pk_halt();
+        }
+
+        //Set LMCR for each CME
+        //If OWNER is switched to CHAR, the last LMCR setting is retained
+        if (G_pgpe_pstate_record.pmcrOwner == PMCR_OWNER_HOST)
+        {
+            p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_HOST);
+        }
+        else if (G_pgpe_pstate_record.pmcrOwner == PMCR_OWNER_OCC)
+        {
+            p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
+        }
+
+        //We set PCB TYPE1 and LMCR for characterization mode
+        //and explicitly set PMCR OWNER to CHAR
+    }
+    else
+    {
+        p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
+        PK_TRACE_DBG("PS: OWNER_CHAR");
+        G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_CHAR;
+        g_oimr_override &= ~(BIT64(46));
+        out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
+    }
+
+    //4. Set External VRM and Send DB0 to every active CME
+    external_voltage_control_init(&G_pgpe_pstate_record.eVidCurr);
+    PK_TRACE_INF("PS: eVidCurr=%umV", G_pgpe_pstate_record.eVidCurr);
+#if SIMICS_TUNING == 1
+    G_pgpe_pstate_record.eVidCurr = p9_pgpe_gppb_intp_vdd_from_ps(G_pgpe_pstate_record.globalPSTarget,
+                                    VPD_PT_SET_BIASED_SYSP, VPD_SLOPES_BIASED);
+#endif
+    G_pgpe_pstate_record.eVidNext = p9_pgpe_gppb_intp_vdd_from_ps(G_pgpe_pstate_record.globalPSTarget,
+                                    VPD_PT_SET_BIASED_SYSP, VPD_SLOPES_BIASED);
+    PK_TRACE_INF("PS: eVidNext=%umV", G_pgpe_pstate_record.eVidNext);
+
+    pgpe_db0_start_ps_bcast_t db0;
+    db0.value = 0;
+    db0.fields.msg_id = MSGID_DB0_START_PSTATE_BROADCAST;
+    db0.fields.global_actual = G_pgpe_pstate_record.globalPSTarget;
+    db0.fields.quad0_ps = G_pgpe_pstate_record.quadPSTarget[0];
+    db0.fields.quad1_ps = G_pgpe_pstate_record.quadPSTarget[1];
+    db0.fields.quad2_ps = G_pgpe_pstate_record.quadPSTarget[2];
+    db0.fields.quad3_ps = G_pgpe_pstate_record.quadPSTarget[3];
+    db0.fields.quad4_ps = G_pgpe_pstate_record.quadPSTarget[4];
+    db0.fields.quad5_ps = G_pgpe_pstate_record.quadPSTarget[5];
+
+    if (G_pgpe_pstate_record.eVidCurr > G_pgpe_pstate_record.eVidNext)
+    {
+        //Send DB0
+        PK_TRACE_DBG("PS: Send DB0");
+
+        for (q = 0; q < MAX_QUADS; q++)
+        {
+            if(G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+            {
+                for (c = q << 2; c < ((q + 1) << 2); c++)
+                {
+                    if (ccsr.value & ((0xc0000000) >> c))
+                    {
+                        p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB0, c), db0.value);
+                    }
+                }
+            }
+        }
+
+        p9_pgpe_wait_cme_db_ack(G_pgpe_pstate_record.quadsActive);//Wait for ACKs from all QuadManagers
+        PK_TRACE_DBG("PS: DB0 ACK");
+
+        p9_pgpe_pstate_updt_ext_volt(G_pgpe_pstate_record.eVidNext); //update voltage
+    }
+    else
+    {
+        p9_pgpe_pstate_updt_ext_volt(G_pgpe_pstate_record.eVidNext); //update voltage
+
+        //Send DB0
+        for (q = 0; q < MAX_QUADS; q++)
+        {
+            if (G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+            {
+                for (c = q << 2; c < ((q + 1) << 2); c++)
+                {
+                    if (ccsr.value & ((0xc0000000) >> c))
+                    {
+                        p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB0, c), db0.value);
+                    }
+                }
+            }
+        }
+
+        PK_TRACE_DBG("PS: Send START DB0");
+        p9_pgpe_wait_cme_db_ack(G_pgpe_pstate_record.quadsActive);//Wait for ACKs from all QuadManagers
+        PK_TRACE_DBG("PS: DB0 ACK");
+    }
+
+    G_pgpe_pstate_record.globalPSCurr = G_pgpe_pstate_record.globalPSTarget;
+    PK_TRACE_INF("PS: GlbPSCurr:0x%x", G_pgpe_pstate_record.globalPSCurr );
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSTarget[q];
+    }
+
+    //Update Shared SRAM
+    G_pgpe_pstate_record.pQuadState0->fields.quad0_pstate = G_pgpe_pstate_record.quadPSCurr[0];
+    G_pgpe_pstate_record.pQuadState0->fields.quad1_pstate = G_pgpe_pstate_record.quadPSCurr[1];
+    G_pgpe_pstate_record.pQuadState0->fields.quad2_pstate = G_pgpe_pstate_record.quadPSCurr[2];
+    G_pgpe_pstate_record.pQuadState0->fields.quad3_pstate = G_pgpe_pstate_record.quadPSCurr[3];
+    G_pgpe_pstate_record.pQuadState1->fields.quad4_pstate = G_pgpe_pstate_record.quadPSCurr[4];
+    G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = G_pgpe_pstate_record.quadPSCurr[5];
+
+
+    //6. Enable PStates
+    G_pgpe_pstate_record.pstatesStatus = PSTATE_ACTIVE;
+    uint32_t occScr2 = in32(OCB_OCCS2);
+    occScr2 |= BIT32(PGPE_PSTATE_PROTOCOL_ACTIVE);
+    PK_TRACE_INF("PS: PGPE_PSTATE_PROTOCOL_ACTIVE set");
+    out32(OCB_OCCS2, occScr2);
+
+    uint32_t opit4pr;
+    opit4pr = in32(OCB_OPIT4PRA);
+    out32(OCB_OPIT4PRA_CLR, opit4pr);
+    out32(OCB_OISR1_CLR, BIT32(17));
+    g_oimr_override &= ~BIT64(49);
+    out32(OCB_OIMR1_CLR, BIT32(17)); //Enable PCB_INTR_TYPE4
+
+    //7. Send Pstate Start ACK to OCC
+    if (pstate_start_origin == PSTATE_START_OCC_IPC)
+    {
+        ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+        ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+        args->msg_cb.rc = PGPE_RC_SUCCESS;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+
+    }
+
+    PK_TRACE_DBG("PS: Start Done");
+}
+
+//
+//p9_pgpe_pstate_stop()
+//
+void p9_pgpe_pstate_stop()
+{
+    PK_TRACE_DBG("PS: Stop Enter");
+    uint32_t q, c;
+    //PkMachineContext  ctx;
+    ocb_ccsr_t ccsr;
+    ccsr.value = in32(OCB_CCSR);
+
+    pgpe_db0_stop_ps_bcast_t db0_stop;
+    db0_stop.value = 0;
+    db0_stop.fields.msg_id = MSGID_DB0_STOP_PSTATE_BROADCAST;
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if (G_pgpe_pstate_record.quadsActive & (0x80 >> q))
+        {
+            for (c = q << 2; c < ((q + 1) << 2); c++)
+            {
+                if (ccsr.value & ((0xc0000000) >> c))
+                {
+                    p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB0, c), db0_stop.value);
+                }
+            }
+        }
+    }
+
+    PK_TRACE_INF("PS: Sent STOP DB0");
+
+    p9_pgpe_wait_cme_db_ack(G_pgpe_pstate_record.quadsActive);//Wait for ACKs from all CMEs
+
+    G_pgpe_pstate_record.pstatesStatus = PSTATE_STOPPED;
+
+    //Send STOP ACK to OCC
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+    ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+    args->msg_cb.rc = PGPE_RC_SUCCESS;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+    ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+
+    PK_TRACE_DBG("PS: Stop Done");
+}
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
index c46db0e0..6d4bfb7f 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
@@ -34,15 +34,14 @@
 
 
 #define MAX_IPC_PEND_TBL_ENTRIES                9
-#define IPC_PEND_PSTATE_START                   0
-#define IPC_PEND_PSTATE_STOP                    1
-#define IPC_PEND_SGPE_ACTIVE_CORES_UPDT         2
-#define IPC_PEND_SGPE_ACTIVE_QUADS_UPDT         3
-#define IPC_PEND_CLIP_UPDT                      4
-#define IPC_PEND_WOF_CTRL                       5
-#define IPC_PEND_WOF_VFRT                       6
-#define IPC_PEND_SET_PMCR_REQ                   7
-#define IPC_PEND_SGPE_SUSPEND_PSTATES           8
+#define IPC_PEND_PSTATE_START_STOP              0
+#define IPC_PEND_SGPE_ACTIVE_CORES_UPDT         1
+#define IPC_PEND_SGPE_ACTIVE_QUADS_UPDT         2
+#define IPC_PEND_CLIP_UPDT                      3
+#define IPC_PEND_WOF_CTRL                       4
+#define IPC_PEND_WOF_VFRT                       5
+#define IPC_PEND_SET_PMCR_REQ                   6
+#define IPC_PEND_SGPE_SUSPEND_PSTATES           7
 
 #define ALL_QUADS_BIT_MASK      QUAD0_BIT_MASK | QUAD1_BIT_MASK | \
     QUAD2_BIT_MASK | QUAD3_BIT_MASK | \
@@ -57,14 +56,14 @@
 
 enum PSTATE_STATUS
 {
-    PSTATE_INIT                                 =    0,
-    PSTATE_START_PENDING                        =    1,
-    PSTATE_ACTIVE                               =    2,
-    PSTATE_STOPPED                              =    3,
-    PSTATE_SUSPENDED                            =    4,
-    PSTATE_PM_SUSPEND_PENDING                   =    5,
-    PSTATE_PM_SUSPENDED                         =    6,
-    PSTATE_SAFE_MODE                            =    7
+    PSTATE_INIT                                 =    0, //PGPE Booted
+    PSTATE_ACTIVE                               =    1, //Pstates are active
+    PSTATE_STOPPED                              =    2, //Pstates are stopped
+    PSTATE_SUSPENDED_WHILE_STOPPED_INIT         =    3, //Suspended by SGPE IPC
+    PSTATE_SUSPENDED_WHILE_ACTIVE               =    4, //Suspended by SGPE IPC
+    PSTATE_PM_SUSPEND_PENDING                   =    5, //PM Complex Suspend Pending
+    PSTATE_PM_SUSPENDED                         =    6, //PM Complex Suspend
+    PSTATE_SAFE_MODE                            =    7  //Safe Mode
 };
 
 //Task list entry
@@ -100,6 +99,8 @@ typedef struct
     uint8_t quadPSNext[MAX_QUADS];      //target Pstate per quad
     uint8_t globalPSNext;
     uint8_t pad3;
+    uint8_t quadPSAtStop11[MAX_QUADS];  //target Pstate per quad
+    uint8_t pad4[2];
     uint16_t alreadySemPosted;
     uint32_t eVidCurr, eVidNext;
     ipc_req_t ipcPendTbl[MAX_IPC_PEND_TBL_ENTRIES];
@@ -110,6 +111,7 @@ typedef struct
     PkSemaphore sem_process_req;
     PkSemaphore sem_actuate;
     PkSemaphore sem_sgpe_wait;
+    uint32_t quadsActive;
 } PgpePstateRecord;
 
 
@@ -129,8 +131,12 @@ void p9_pgpe_pstate_safe_mode();
 void p9_pgpe_pstate_apply_safe_clips();
 int32_t p9_pgpe_pstate_at_target();
 void p9_pgpe_pstate_do_step();
+//void p9_pgpe_pstate_do_suspend();
 void p9_pgpe_pstate_set_pmcr_owner(uint32_t owner);
-void p9_pgpe_wait_cme_db_ack(uint8_t msg_id, uint32_t activeCores);
+void p9_pgpe_wait_cme_db_ack(uint32_t activeCores);
 void p9_pgpe_pstate_updt_ext_volt(uint32_t tgtEVid);
-
+void p9_pgpe_pstate_process_quad_entry(uint32_t quadsAffected);
+void p9_pgpe_pstate_process_quad_exit(uint32_t quadsAffected);
+void p9_pgpe_pstate_start(uint32_t pstate_start_origin);
+void p9_pgpe_pstate_stop();
 #endif //
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
index 9f9bef73..2c5677c0 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
@@ -35,13 +35,7 @@
 #include "avs_driver.h"
 
 
-//#defines
-#define PSTATE_START_OCC_FLAG 0
-#define PSTATE_START_OCC_IPC  1
-
 //Local Function Prototypes
-void p9_pgpe_thread_actuate_start(uint32_t pstate_start_origin);
-void p9_pgpe_thread_actuate_stop();
 void p9_pgpe_thread_actuate_init_actual_quad();
 
 //
@@ -62,8 +56,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
     uint32_t inRange, q  = 0;
     PkMachineContext  ctx;
     uint32_t restore_irq = 0;
-    ocb_qcsr_t qcsr;
-    qcsr.value = in32(OCB_QCSR);
 
     pk_semaphore_create(&(G_pgpe_pstate_record.sem_actuate), 0, 1);
     pk_semaphore_create(&(G_pgpe_pstate_record.sem_sgpe_wait), 0, 1);
@@ -87,7 +79,7 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
             G_pgpe_pstate_record.psClipMin[q] = G_gppb->operating_points[ULTRA].pstate;
         }
 
-        p9_pgpe_thread_actuate_start(PSTATE_START_OCC_FLAG);
+        p9_pgpe_pstate_start(PSTATE_START_OCC_FLAG);
     }
 
     // Set OCC Scratch2[PGPE_ACTIVE]
@@ -106,8 +98,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
             pk_semaphore_pend(&(G_pgpe_pstate_record.sem_actuate), PK_WAIT_FOREVER);
             wrteei(1);
 
-            //Pstates Start. This call will unmask IPC and block on SGPE ACK
-            p9_pgpe_thread_actuate_start(PSTATE_START_OCC_IPC);
         }
 
         //Loop while Pstate is enabled
@@ -115,14 +105,25 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
 
         restore_irq = 0;
 
-        while(G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE || G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED)
+        while(G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE
+              || G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_ACTIVE)
         {
-            if(G_pgpe_pstate_record.pstatesStatus != PSTATE_SUSPENDED)
+            if(G_pgpe_pstate_record.pstatesStatus != PSTATE_SUSPENDED_WHILE_ACTIVE)
             {
                 //Actuate step(if needed)
                 if(p9_pgpe_pstate_at_target() == 0)
                 {
-                    p9_pgpe_pstate_do_step();
+                    //We check that pstates are active after entering critical section
+                    //It's possible that some IPC which is processed by PROCESS THREAD
+                    //has updated the PstatesState to a !PSTATE_ACTIVE state
+                    pk_irq_sub_critical_enter(&ctx);
+
+                    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
+                    {
+                        p9_pgpe_pstate_do_step();
+                    }
+
+                    pk_irq_sub_critical_exit(&ctx);
                 }
 
                 //Check if CLIP_UPDT is pending and Pstates are clipped
@@ -131,13 +132,11 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                     (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 1))
                 {
                     inRange = 1;
-
                     uint32_t minPS;
 
                     for (q = 0; q < MAX_QUADS; q++)
                     {
-                        if((G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q)) &&
-                           (qcsr.fields.ex_config & (0xC00 >> (2 * q))))
+                        if((G_pgpe_pstate_record.quadsActive & (0x80 >> q)))
                         {
                             minPS = G_pgpe_pstate_record.psClipMin[q];
 
@@ -153,7 +152,6 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                             if (G_pgpe_pstate_record.quadPSCurr[q] > G_pgpe_pstate_record.psClipMax[q] ||
                                 G_pgpe_pstate_record.quadPSCurr[q] <  minPS)
                             {
-                                //            PK_TRACE_DBG("ACT_TH:!Clipped[%d) qPSCur: 0x%x", q, G_pgpe_pstate_record.quadPSCurr[q]);
                                 inRange = 0;
                             }
                         }
@@ -191,11 +189,9 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                             ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].cmd, IPC_RC_SUCCESS);
                         }
 
-
-
                         restore_irq = 1;
                     }
-                }
+                } //Pending ACKs
 
                 //Check if IPC should be opened again
                 if (restore_irq == 1)
@@ -205,27 +201,33 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
                     pk_irq_vec_restore(&ctx);
                 }
             }
-        } //while loop
+            else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_ACTIVE)
+            {
+                pk_semaphore_pend(&(G_pgpe_pstate_record.sem_actuate), PK_WAIT_FOREVER);
+            }
+        } //while ACTIVE or SUSPENDED loop
 
-        if (G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED)
-        {
-            //Pstates Stop
-            //If we entered SAFE_MODE already, then do STOP protocol
-            //This call will unmask IPC and block on SGPE ACK
-            p9_pgpe_thread_actuate_stop();
-        }
-        else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING
-                 || G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE)
+        /*        if (G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED)
+                {
+                    //Pstates Stop
+                    //If we entered SAFE_MODE already, then do STOP protocol
+                    //This call will unmask IPC and block on SGPE ACK
+                    p9_pgpe_thread_actuate_stop();
+                }*/
+        if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING
+            || G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE)
         {
             //Actuate to PSAFE Pstate
             while (p9_pgpe_pstate_at_target() == 0)
             {
+                pk_irq_sub_critical_enter(&ctx);
                 p9_pgpe_pstate_do_step();
+                pk_irq_sub_critical_exit(&ctx);
             }
 
             if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING)
             {
-                p9_pgpe_pstate_send_suspend_stop(); //Notify SGPE}
+                p9_pgpe_pstate_send_suspend_stop(); //Notify SGPE
             }
             else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE)
             {
@@ -237,384 +239,7 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
     }//Thread loop
 }
 
-//
-//p9_pgpe_thread_actuate_start()
-//
-void p9_pgpe_thread_actuate_start(uint32_t pstate_start_origin)
-{
-    PK_TRACE_DBG("ACT_TH: Start Enter");
-    ocb_ccsr_t ccsr;
-    ccsr.value = in32(OCB_CCSR);
-    ocb_qcsr_t qcsr;
-    qcsr.value = in32(OCB_QCSR);
-    uint8_t quadPstates[MAX_QUADS];
-    uint32_t lowestDpll, syncPstate;
-    qppm_dpll_stat_t dpll;
-    PkMachineContext  ctx;
-    uint32_t active_conf_cores = 0;
-    uint32_t q;
-
-    G_pgpe_pstate_record.pstatesStatus = PSTATE_START_PENDING;
-
-    //1. Send 'Control Stop Updates' IPC to SGPE
-#if SGPE_IPC_ENABLED == 1
-    uint32_t rc;
-    G_sgpe_control_updt.fields.msg_num = MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES;
-    G_sgpe_control_updt.fields.return_code = 0x0;
-    G_sgpe_control_updt.fields.action = CTRL_STOP_UPDT_ENABLE_QUAD;
-    G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
-    ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
-                 IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
-                 p9_pgpe_pstate_ipc_rsp_cb_sem_post,
-                 (void*)&G_pgpe_pstate_record.sem_sgpe_wait);
-
-    //send the command
-    PK_TRACE_DBG("ACT_TH: CtrlStopUpdt Sent");
-    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
-
-    if(rc)
-    {
-        pk_halt();
-    }
-
-    //Wait for SGPE ACK with alive Quads
-    pk_irq_vec_restore(&ctx);
-    pk_semaphore_pend(&(G_pgpe_pstate_record.sem_sgpe_wait), PK_WAIT_FOREVER);
-
-    PK_TRACE_DBG("ACT_TH: CtrlStopUpdt Resp");
-
-    if (G_sgpe_control_updt.fields.return_code == SGPE_PGPE_IPC_RC_SUCCESS)
-    {
-        //Update Shared Memory Region
-        PK_TRACE_DBG("ACT_TH: ctrl_updt=0x%08x%08x", G_sgpe_control_updt.value >> 32, G_sgpe_control_updt.value);
-        G_pgpe_pstate_record.pQuadState0->fields.active_cores = (ccsr.value >> 16);
-        G_pgpe_pstate_record.pQuadState1->fields.active_cores = ccsr.value & 0xFF00 ;
-        G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = G_sgpe_control_updt.fields.active_quads << 2;
-        PK_TRACE_DBG("ACT_TH: core_pwr_st0=0x%x", (uint32_t)G_pgpe_pstate_record.pQuadState0->fields.active_cores);
-        PK_TRACE_DBG("ACT_TH: core_pwr_st1=0x%x", (uint32_t)G_pgpe_pstate_record.pQuadState1->fields.active_cores);
-        PK_TRACE_DBG("ACT_TH: req_active_quad=0x%x",
-                     (uint32_t)G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads);
-    }
-    else
-    {
-        PK_TRACE_DBG("ACT_TH: SGPE Ctrl Stop rc=0x%x", (uint32_t)G_sgpe_control_updt.fields.return_code);
-        pk_halt();
-    }
-
-#else
-    G_pgpe_pstate_record.pQuadState0->fields.active_cores = (ccsr.value >> 16);
-    G_pgpe_pstate_record.pQuadState1->fields.active_cores = (ccsr.value & 0xFF00);
-
-    for (q = 0; q < MAX_QUADS; q++ )
-    {
-        if ((qcsr.fields.ex_config & (0x800 >> (2 * q))) ||
-            (qcsr.fields.ex_config & (0x400 >> (2 * q))))
-        {
-            G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (0x80 >> q) ;
-        }
-    }
-
-#endif //_SGPE_IPC_ENABLED_
-
-    PK_TRACE_DBG("ACT_TH: Shr Mem Updt");
-
-    //2. Read DPLLs
-    lowestDpll = 0xFFF;
-    dpll.value = 0;
-    PK_TRACE_INF("ACT_TH: DPLL0Value=0x%x", G_gppb->dpll_pstate0_value);
-
-    for (q = 0; q < MAX_QUADS; q++)
-    {
-        if((G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q)) &&
-           (qcsr.fields.ex_config & (0xC00 >> 2 * q)))
-        {
-#if SIMICS_TUNING == 0
-            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_STAT, q), dpll.value);
-#else
-            dpll.fields.freqout = (G_gppb->dpll_pstate0_value
-                                   - G_gppb->operating_points_set[VPD_PT_SET_BIASED_SYSP][NOMINAL].pstate);
-#endif
-
-            if ((dpll.fields.freqout)  < lowestDpll )
-            {
-                lowestDpll = dpll.fields.freqout;
-            }
-
-            PK_TRACE_DBG("ACT_TH: Quad[%d]: DPLL=0x%x", q, (dpll.fields.freqout));
-            active_conf_cores |= (0xF0000000 >> q * 4);
-        }
-    }
-
-    //3. Determine Sync Pstate
-    //
-    //\TODO: Need Greg's Response
-    //Why does HCode Spec says current frequency might not
-    //correspond to frequency of any Pstate? As I understand,
-    //DPLL encoding and frequency/pstate have same step size, so
-    //they should correspond one to one
-    //
-    if (lowestDpll > G_gppb->dpll_pstate0_value)
-    {
-        syncPstate = G_gppb->dpll_pstate0_value;
-    }
-    else
-    {
-        syncPstate = G_gppb->dpll_pstate0_value - lowestDpll;
-    }
-
-    //Init Core PStates
-    for (q = 0; q < MAX_QUADS; q++)
-    {
-        if((qcsr.fields.ex_config & (0xC00 >> 2 * q)))
-        {
-            quadPstates[q] = syncPstate;
-        }
-        else
-        {
-            quadPstates[q] = 0xFF;
-        }
-    }
-
-    p9_pgpe_pstate_update(quadPstates);
-    PK_TRACE_INF("ACT_TH: Sync Pstate=0x%x", syncPstate);
-    PK_TRACE_INF("ACT_TH: LowDPLL:0x%x", lowestDpll);
-    PK_TRACE_INF("ACT_TH: GlblPsTgt:0x%x", G_pgpe_pstate_record.globalPSTarget);
-
-    //4. Determine PMCR Owner
-    //We save it off so that CMEs that are currently in STOP11
-    //can be told upon STOP11 Exit
-    if (pstate_start_origin == PSTATE_START_OCC_IPC)
-    {
-        PK_TRACE_DBG("ACT_TH: g_oimr_override:0x%llx", g_oimr_override);
-        ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].cmd;
-        ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
-
-        if (args->pmcr_owner == PMCR_OWNER_HOST)
-        {
-            PK_TRACE_DBG("ACT_TH: OWNER_HOST");
-            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_HOST;
-            g_oimr_override &= ~(BIT64(46));
-            out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
-        }
-        else if (args->pmcr_owner == PMCR_OWNER_OCC)
-        {
-            PK_TRACE_DBG("ACT_TH: OWNER_OCC");
-            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_OCC;
-            g_oimr_override |= BIT64(46);
-            out32(OCB_OIMR1_OR, BIT32(14)); //Disable PCB_INTR_TYPE1
-        }
-        else if (args->pmcr_owner == PMCR_OWNER_CHAR)
-        {
-            PK_TRACE_DBG("ACT_TH: OWNER_CHAR");
-            G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_CHAR;
-            g_oimr_override &= ~(BIT64(46));
-            out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
-        }
-        else
-        {
-            pk_halt();
-        }
-
-        //Set LMCR for each CME
-        //If OWNER is switched to CHAR, the last LMCR setting is retained
-        if (G_pgpe_pstate_record.pmcrOwner == PMCR_OWNER_HOST)
-        {
-            p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_HOST);
-        }
-        else if (G_pgpe_pstate_record.pmcrOwner == PMCR_OWNER_OCC)
-        {
-            p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
-        }
-
-        //We set PCB TYPE1 and LMCR for characterization mode
-        //and explicitly set PMCR OWNER to CHAR
-    }
-    else
-    {
-        p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
-        PK_TRACE_DBG("ACT_TH: OWNER_CHAR");
-        G_pgpe_pstate_record.pmcrOwner = PMCR_OWNER_CHAR;
-        g_oimr_override &= ~(BIT64(46));
-        out32(OCB_OIMR1_CLR, BIT32(14)); //Enable PCB_INTR_TYPE1
-    }
-
-    //4. Set External VRM and Send DB0 to every active CME
-    //\TODO: Need Greg's Response
-    //Why does spec says to read external VDD. Is there
-    //any other reason besides PGPE storing it
-    external_voltage_control_init(&G_pgpe_pstate_record.eVidCurr);
-    PK_TRACE_INF("ACT_TH: eVidCurr=%umV", G_pgpe_pstate_record.eVidCurr);
-#if SIMICS_TUNING == 1
-    G_pgpe_pstate_record.eVidCurr = p9_pgpe_gppb_intp_vdd_from_ps(G_pgpe_pstate_record.globalPSTarget,
-                                    VPD_PT_SET_BIASED_SYSP, VPD_SLOPES_BIASED);
-#endif
-    G_pgpe_pstate_record.eVidNext = p9_pgpe_gppb_intp_vdd_from_ps(G_pgpe_pstate_record.globalPSTarget,
-                                    VPD_PT_SET_BIASED_SYSP, VPD_SLOPES_BIASED);
-    PK_TRACE_INF("ACT_TH: eVidNext=%umV", G_pgpe_pstate_record.eVidNext);
-
-    pgpe_db0_start_ps_bcast_t db0;
-    db0.value = 0;
-    db0.fields.msg_id = MSGID_DB0_START_PSTATE_BROADCAST;
-    db0.fields.global_actual = G_pgpe_pstate_record.globalPSTarget;
-    db0.fields.quad0_ps = G_pgpe_pstate_record.quadPSTarget[0];
-    db0.fields.quad1_ps = G_pgpe_pstate_record.quadPSTarget[1];
-    db0.fields.quad2_ps = G_pgpe_pstate_record.quadPSTarget[2];
-    db0.fields.quad3_ps = G_pgpe_pstate_record.quadPSTarget[3];
-    db0.fields.quad4_ps = G_pgpe_pstate_record.quadPSTarget[4];
-    db0.fields.quad5_ps = G_pgpe_pstate_record.quadPSTarget[5];
-
-    if (G_pgpe_pstate_record.eVidCurr > G_pgpe_pstate_record.eVidNext)
-    {
-        p9_dd1_db_multicast_wr(PCB_MUTLICAST_GRP1 | CPPM_CMEDB0, db0.value, active_conf_cores);
-        PK_TRACE_DBG("ACT_TH: Send DB0");
-        p9_pgpe_wait_cme_db_ack(MSGID_DB0_START_PSTATE_BROADCAST, active_conf_cores);//Wait for ACKs from all QuadManagers
-        PK_TRACE_DBG("ACT_TH: DB0 ACK");
-        p9_pgpe_pstate_updt_ext_volt(G_pgpe_pstate_record.eVidNext);
-    }
-    else
-    {
-        p9_pgpe_pstate_updt_ext_volt(G_pgpe_pstate_record.eVidNext);
-        p9_dd1_db_multicast_wr(PCB_MUTLICAST_GRP1 | CPPM_CMEDB0, db0.value, active_conf_cores);
-        PK_TRACE_DBG("ACT_TH: Send DB0");
-        p9_pgpe_wait_cme_db_ack(MSGID_DB0_START_PSTATE_BROADCAST, active_conf_cores);//Wait for ACKs from all QuadManagers
-        PK_TRACE_DBG("ACT_TH: DB0 ACK");
-    }
-
-    G_pgpe_pstate_record.globalPSCurr = G_pgpe_pstate_record.globalPSTarget;
-    PK_TRACE_INF("ACT_TH: GlbPSCurr:0x%x", G_pgpe_pstate_record.globalPSCurr );
-
-    for (q = 0; q < MAX_QUADS; q++)
-    {
-        G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSTarget[q];
-    }
-
-    //Update Shared SRAM
-    G_pgpe_pstate_record.pQuadState0->fields.quad0_pstate = G_pgpe_pstate_record.quadPSCurr[0];
-    G_pgpe_pstate_record.pQuadState0->fields.quad1_pstate = G_pgpe_pstate_record.quadPSCurr[1];
-    G_pgpe_pstate_record.pQuadState0->fields.quad2_pstate = G_pgpe_pstate_record.quadPSCurr[2];
-    G_pgpe_pstate_record.pQuadState0->fields.quad3_pstate = G_pgpe_pstate_record.quadPSCurr[3];
-    G_pgpe_pstate_record.pQuadState1->fields.quad4_pstate = G_pgpe_pstate_record.quadPSCurr[4];
-    G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = G_pgpe_pstate_record.quadPSCurr[5];
-
-
-    //7. Enable PStates
-    G_pgpe_pstate_record.pstatesStatus = PSTATE_ACTIVE;
-    PK_TRACE_DBG("ACT_TH: PSTATE_ACTIVE");
-
-    //8. Send Pstate Start ACK to OCC
-    if (pstate_start_origin == PSTATE_START_OCC_IPC)
-    {
-        ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].cmd;
-        ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
-        args->msg_cb.rc = PGPE_RC_SUCCESS;
-        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].pending_ack = 0;
-        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].cmd, IPC_RC_SUCCESS);
-
-        if ((G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing == 1) ||
-            (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing == 1))
-        {
-            PK_TRACE_DBG("ACT_TH: Post PROC_TH");
-            pk_semaphore_post(&G_pgpe_pstate_record.sem_process_req);
-        }
-        else
-        {
-            PK_TRACE_DBG("ACT_TH: Restoring IRQs");
-            pk_irq_vec_restore(&ctx);
-        }
-    }
-
-    PK_TRACE_DBG("ACT_TH: Start Exit");
-}
-
-//
-//p9_pgpe_thread_actuate_stop()
-//
-void p9_pgpe_thread_actuate_stop()
-{
-    PK_TRACE_DBG("ACT_TH: Stop Enter");
-    PK_TRACE_DBG("ACT_TH: GlbPSCurr:0x%x", G_pgpe_pstate_record.globalPSCurr );
-    PK_TRACE_DBG("ACT_TH: GlblPsTgt:0x%x", G_pgpe_pstate_record.globalPSTarget);
-    uint32_t q;
-    PkMachineContext  ctx;
-    uint32_t active_conf_cores = 0;
-    ocb_qcsr_t qcsr;
-    qcsr.value = in32(OCB_QCSR);
-
-    //Determine active and configured cores
-    for (q = 0; q < MAX_QUADS; q++)
-    {
-        if(G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q))
-        {
-            if (qcsr.fields.ex_config & (0x800 >> 2 * q))
-            {
-                active_conf_cores |= (0xC0000000 >> q * 4);
-            }
-
-            if (qcsr.fields.ex_config & (0x400 >> 2 * q))
-            {
-                active_conf_cores |= (0x30000000 >> q * 4);
-            }
-        }
-    }
-
-    pgpe_db0_stop_ps_bcast_t db0_stop;
-    db0_stop.value = 0;
-    db0_stop.fields.msg_id = MSGID_DB0_STOP_PSTATE_BROADCAST;
-    p9_dd1_db_multicast_wr(PCB_MUTLICAST_GRP1 | CPPM_CMEDB0, db0_stop.value, active_conf_cores);
-
-    //Wait for ACKs from all CMEs
-    p9_pgpe_wait_cme_db_ack(MSGID_DB0_STOP_PSTATE_BROADCAST, active_conf_cores);
-
-#if SGPE_IPC_ENABLED == 1
-    uint32_t rc;
-    //Send "Disable Core & Quad Stop Updates" IPC to SGPE
-    G_sgpe_control_updt.fields.return_code = 0x0;
-    G_sgpe_control_updt.fields.action = CTRL_STOP_UPDT_DISABLE_CORE_QUAD;
-    G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
-    ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
-                 IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
-                 p9_pgpe_pstate_ipc_rsp_cb_sem_post,
-                 (void*)&G_pgpe_pstate_record.sem_sgpe_wait);
-
-    //send the command
-    PK_TRACE_DBG("ACT_TH: CtrlStopUpdt Sent");
-    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
-
-    if(rc)
-    {
-        pk_halt();
-    }
-
-    //Wait for SGPE ACK with alive Quads
-    pk_irq_vec_restore(&ctx);
-    pk_semaphore_pend(&(G_pgpe_pstate_record.sem_sgpe_wait), PK_WAIT_FOREVER);
-
-    PK_TRACE_DBG("ACT_TH: CtrlStopUpdt Rcvd");
-
-    if (G_sgpe_control_updt.fields.return_code != SGPE_PGPE_IPC_RC_SUCCESS)
-    {
-        pk_halt();
-    }
-
-#endif// _SGPE_IPC_ENABLED_
-
-    //Send STOP ACK to OCC
-    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_STOP].cmd;
-    ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
-    args->msg_cb.rc = PGPE_RC_SUCCESS;
-    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_STOP].pending_ack = 0;
-    ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_STOP].cmd, IPC_RC_SUCCESS);
-
-    pk_irq_vec_restore(&ctx);
-    PK_TRACE_DBG("ACT_TH: Stop Exit");
-}
 
-//
-//p9_pgpe_thread_actuate_do_step
-//
-//void p9_pgpe_thread_actuate_do_step()
-//{
-//}
 
 void p9_pgpe_thread_actuate_init_actual_quad()
 {
@@ -628,4 +253,5 @@ void p9_pgpe_thread_actuate_init_actual_quad()
     G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = 0xff;
     G_pgpe_pstate_record.pQuadState1->fields.active_cores = 0x0;
     G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = 0x0;
+
 }
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
index a8504fa7..af0c19a8 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
@@ -44,6 +44,7 @@ extern ipc_async_cmd_t G_ipc_msg_pgpe_sgpe;
 //
 void p9_pgpe_process_sgpe_updt_active_cores();
 void p9_pgpe_process_sgpe_updt_active_quads();
+void p9_pgpe_process_start_stop();
 void p9_pgpe_process_sgpe_suspend_pstates();
 void p9_pgpe_process_clip_updt();
 void p9_pgpe_process_wof_ctrl();
@@ -79,9 +80,10 @@ void p9_pgpe_thread_process_requests(void* arg)
         wrteei(1);
 
         //Enter Sub-Critical Section. Timer Interrupts are enabled
-        pk_critical_section_enter(&ctx);
+        /*pk_critical_section_enter(&ctx);
         pk_irq_save_and_set_mask(0);
-        pk_critical_section_exit(&ctx);
+        pk_critical_section_exit(&ctx);*/
+        pk_irq_sub_critical_enter(&ctx);
 
         G_pgpe_pstate_record.alreadySemPosted  = 0;
         restore_irq  = 1;
@@ -89,7 +91,7 @@ void p9_pgpe_thread_process_requests(void* arg)
         PK_TRACE_DBG("PROCTH: Process Task\n");
 
         //Go through IPC Pending Table
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing)
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing == 1)
         {
             p9_pgpe_process_sgpe_updt_active_cores();
 
@@ -99,7 +101,7 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing)
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing == 1)
         {
             p9_pgpe_process_sgpe_updt_active_quads();
 
@@ -109,7 +111,17 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_processing)
+        if(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing == 1)
+        {
+            p9_pgpe_process_start_stop();
+
+            /*if(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].pending_ack == 1 ||
+               G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_STOP].pending_ack == 1) {
+                restore_irq = 0;
+            }*/
+        }
+
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_processing == 1)
         {
             p9_pgpe_process_sgpe_suspend_pstates();
 
@@ -119,7 +131,7 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_processing)
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_processing == 1)
         {
             p9_pgpe_process_clip_updt();
 
@@ -139,7 +151,7 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_processing)
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_processing == 1)
         {
             p9_pgpe_process_wof_vfrt();
 
@@ -149,7 +161,7 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SET_PMCR_REQ].pending_processing)
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SET_PMCR_REQ].pending_processing == 1)
         {
             p9_pgpe_process_set_pmcr_req();
 
@@ -159,15 +171,8 @@ void p9_pgpe_thread_process_requests(void* arg)
             }
         }
 
-        //Pstate START/STOP IPCs are processed and acked in actuate thread,
-        //so don't call pk_irq_vec_restore if they are pending.
-        if(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START].pending_ack == 1 ||
-           G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_STOP].pending_ack == 1 )
-        {
-            restore_irq = 0;
-        }
-
-        pk_irq_vec_restore(&ctx); //End Sub-Critical Section
+        pk_irq_sub_critical_exit(&ctx);
+        //pk_irq_vec_restore(&ctx); //End Sub-Critical Section
 
         //Restore IPC if no pending acks. Otherwise, actuate thread will
         //eventually restore IPC interrupt
@@ -190,36 +195,21 @@ void p9_pgpe_process_sgpe_updt_active_cores()
     ipcmsg_s2p_update_active_cores_t* args = (ipcmsg_s2p_update_active_cores_t*)async_cmd->cmd_data;
 
     //If in PM_SUSPENDED state, then ack back with error
-    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED )
+    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED)
     {
         args->fields.return_code = SGPE_PGPE_RC_PM_COMPLEX_SUSPEND;
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd, IPC_RC_SUCCESS);
         pk_halt();
     }
-    //Active quad updates should only be received if Pstates and WOF are enabled.
-    //However, if we have start pending, then we should not process is here. Once,
-    //actuate thread completes actuate_start, it checks for any pending_processing tasks,
-    //if there are any it post to process thread(this thread). This cmd will get processed
-    //then
-    else if(G_pgpe_pstate_record.pstatesStatus != PSTATE_START_PENDING)
+    else
     {
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_processing = 0;
 
-        if(G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED || G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT
-           || G_pgpe_pstate_record.wofEnabled == 0)
+        if(G_pgpe_pstate_record.wofEnabled == 0)
         {
-            if(G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED || G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT)
-            {
-                PK_TRACE_DBG("PROCTH: C Updt(Pstate !Started)\n");
-                args->fields.return_code = PGPE_RC_PSTATES_NOT_STARTED;
-            }
-            else
-            {
-                PK_TRACE_DBG("PROCTH: C Updt(WOF_Disabled)\n");
-                args->fields.return_code = PGPE_WOF_RC_NOT_ENABLED;
-            }
-
+            PK_TRACE_DBG("PROCTH: C Updt(WOF_Disabled)\n");
+            args->fields.return_code = PGPE_WOF_RC_NOT_ENABLED;
             G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
             ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd, IPC_RC_SUCCESS);
         }
@@ -252,68 +242,140 @@ void p9_pgpe_process_sgpe_updt_active_cores()
 //
 void p9_pgpe_process_sgpe_updt_active_quads()
 {
-    PK_TRACE_DBG("PROCTH: Quad Updt Entry\n");
+    PK_TRACE_DBG("PROCTH: Q Updt Start\n");
 
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd;
     ipcmsg_s2p_update_active_quads_t* args = (ipcmsg_s2p_update_active_quads_t*)async_cmd->cmd_data;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing = 0;
 
     //If in PM_SUSPENDED state, then ack back with error
-    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED )
+    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED ||
+        G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING ||
+        G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE)
     {
         args->fields.return_code = SGPE_PGPE_RC_PM_COMPLEX_SUSPEND;
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack = 0;
         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd, IPC_RC_SUCCESS);
         pk_halt();
     }
-    //Active quad updates should only be received if Pstates are enabled.
-    //However, if we have start pending, then we should not process it here. Once,
-    //actuate thread completes actuate_start, it checks for any pending_processing tasks,
-    //if there are any it post to process thread(this thread). This cmd will get processed
-    //then
-    else if(G_pgpe_pstate_record.pstatesStatus != PSTATE_START_PENDING)
+    else
     {
-        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_processing = 0;
 
-        if(G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT || G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED)
+        //ENTRY
+        if (args->fields.update_type == UPDATE_ACTIVE_TYPE_ENTRY)
         {
-            PK_TRACE_DBG("PROCTH: Q Updt(Pstate Stopped)\n");
-            args->fields.return_code = PGPE_RC_PSTATES_NOT_STARTED;
+            //Update Shared Memory Region
+            PK_TRACE_DBG("PROCTH: Q Updt Entry, Req_Q: 0x%x\n", (uint32_t)(args->fields.requested_quads));
+            G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads &= (~(args->fields.requested_quads << 2));
+
+            //For Entry ACK back to SGPE right away
+            args->fields.return_code = SGPE_PGPE_IPC_RC_SUCCESS;
             G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack = 0;
             ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd, IPC_RC_SUCCESS);
+            PK_TRACE_DBG("PROCTH: Q Updt Entry, ACK sent to SGPE\n");
+
+            //If WOF Enabled, then interlock with OCC
+            if(G_pgpe_pstate_record.wofEnabled == 1)
+            {
+                GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
+            }
+            else
+            {
+                p9_pgpe_pstate_process_quad_entry(args->fields.requested_quads << 2);
+            }
         }
+        //EXIT
         else
         {
             //Update Shared Memory Region
-            PK_TRACE_DBG("PROCTH: Q Updt, Req_Q: 0x%x\n", (uint32_t)(args->fields.requested_quads));
+            PK_TRACE_DBG("PROCTH: Q Updt Exit, Req_Q: 0x%x\n", (uint32_t)(args->fields.requested_quads));
+            G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (args->fields.requested_quads << 2);
 
-            //If ENTRY then ACK to SGPE immediately
-            if (args->fields.update_type == UPDATE_ACTIVE_TYPE_ENTRY)
+            //WOF Enabled
+            if(G_pgpe_pstate_record.wofEnabled == 1)
             {
-                G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads &= (~(args->fields.requested_quads << 2));
-                args->fields.return_code = SGPE_PGPE_IPC_RC_SUCCESS;
-                G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack = 0;
-                ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd, IPC_RC_SUCCESS);
+                GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
             }
             else
             {
-                G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (args->fields.requested_quads << 2);
+                p9_pgpe_pstate_process_quad_exit(args->fields.requested_quads << 2);
             }
+        }
+    }
 
-            if(G_pgpe_pstate_record.wofEnabled == 1)
+    PK_TRACE_DBG("PROCTH: Quad Updt End\n");
+}
+
+void p9_pgpe_process_start_stop()
+{
+    PK_TRACE_DBG("PROCTH: Start/Stop Entry\n");
+    ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd;
+    ipcmsg_start_stop_t* args = (ipcmsg_start_stop_t*)async_cmd->cmd_data;
+
+    if(G_pgpe_header_data->g_pgpe_qm_flags & OCC_IPC_IMMEDIATE_RESP)
+    {
+        PK_TRACE_DBG("START_STOP: Imm\n");
+        args->msg_cb.rc = PGPE_RC_SUCCESS;
+        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+    }
+    else if(G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING ||
+            G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED ||
+            G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE)
+    {
+        PK_TRACE_DBG("START_STOP: PM_SUSP/Safe\n");
+        args->msg_cb.rc = PGPE_RC_PM_COMPLEX_SUSPEND_SAFE_MODE;
+        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+    }
+    else
+    {
+        //If Start
+        if (args->action == PGPE_ACTION_PSTATE_START)
+        {
+            if(G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT || G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED)
             {
-                //Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
-                GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(30));
+                p9_pgpe_pstate_start(PSTATE_START_OCC_IPC);
+                pk_semaphore_post(&G_pgpe_pstate_record.sem_actuate);
             }
-            else
+            else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
+            {
+                p9_pgpe_pstate_set_pmcr_owner(PMCR_OWNER_OCC);
+            }
+            else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_ACTIVE ||
+                     G_pgpe_pstate_record.pstatesStatus ==  PSTATE_SUSPENDED_WHILE_STOPPED_INIT)
+            {
+                PK_TRACE_DBG("START_STOP: Q Entry Pending\n");
+                //do nothing (change ownership upon quad_entry)
+            }
+        }
+        else
+        {
+            if(G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT
+               || G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED
+               || G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_STOPPED_INIT)
+            {
+                PK_TRACE_DBG("START_STOP: Already Stopped\n");
+                args->msg_cb.rc = PGPE_RC_REQ_PSTATE_ALREADY_STOPPED;
+                ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].cmd, IPC_RC_SUCCESS);
+                G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_ack = 0;
+                G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing = 0;
+            }
+            else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
             {
-                //Do auction
-                p9_pgpe_pstate_do_auction(ALL_QUADS_BIT_MASK);
-                p9_pgpe_pstate_apply_clips();
+                p9_pgpe_pstate_stop();
+            }
+            else if (G_pgpe_pstate_record.pstatesStatus == PSTATE_SUSPENDED_WHILE_ACTIVE )
+            {
+                PK_TRACE_DBG("START_STOP: Q Entry Pending\n");
+                //do nothing (pstate stop upon quad entry)
             }
         }
     }
 
-    PK_TRACE_DBG("PROCTH: Quad Updt Exit\n");
+    PK_TRACE_DBG("PROCTH: Start/Stop End\n");
 }
 
 void p9_pgpe_process_sgpe_suspend_pstates()
@@ -321,34 +383,36 @@ void p9_pgpe_process_sgpe_suspend_pstates()
     PK_TRACE_DBG("PROCTH: Susp Pst Entry\n");
     ipc_async_cmd_t* async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].cmd;
     ipcmsg_s2p_suspend_pstate_t* args = (ipcmsg_s2p_suspend_pstate_t*)async_cmd->cmd_data;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_processing = 0;
 
     //If in PM_SUSPENDED state, then ack back with error
-    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED )
+    if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPENDED ||
+        G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING ||
+        G_pgpe_pstate_record.pstatesStatus == PSTATE_SAFE_MODE )
     {
+        PK_TRACE_DBG("PROCTH: Susp Pst while SUSPEND/SAFE\n");
         args->fields.return_code = SGPE_PGPE_RC_PM_COMPLEX_SUSPEND;
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_ack = 0;
         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].cmd, IPC_RC_SUCCESS);
         pk_halt();
-        //Active quad updates should only be received if Pstates are enabled.
-        //However, if we have start pending, then we should not process it here. Once,
-        //actuate thread completes actuate_start, it checks for any pending_processing tasks,
-        //if there are any it posts to process thread(this thread).
-        //This cmd will get processed then
     }
-    else if(G_pgpe_pstate_record.pstatesStatus != PSTATE_START_PENDING)
+    else if(G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT ||
+            G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED ||
+            G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
     {
-        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_processing = 0;
+        args->fields.return_code = SGPE_PGPE_IPC_RC_SUCCESS;
+        G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_ack = 0;
+        ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].cmd, IPC_RC_SUCCESS);
 
-        if(G_pgpe_pstate_record.pstatesStatus == PSTATE_INIT || G_pgpe_pstate_record.pstatesStatus == PSTATE_STOPPED)
+        if(G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
         {
-            PK_TRACE_DBG("PROCTH: Susp Pst(Pstate !Started)\n");
-            args->fields.return_code = PGPE_RC_PSTATES_NOT_STARTED;
-            G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].pending_ack = 0;
-            ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_SUSPEND_PSTATES].cmd, IPC_RC_SUCCESS);
+            PK_TRACE_DBG("PROCTH: Susp Pst while ACTIVE\n");
+            G_pgpe_pstate_record.pstatesStatus = PSTATE_SUSPENDED_WHILE_ACTIVE;
         }
         else
         {
-            G_pgpe_pstate_record.pstatesStatus = PSTATE_SUSPENDED;
+            G_pgpe_pstate_record.pstatesStatus = PSTATE_SUSPENDED_WHILE_STOPPED_INIT;
+            PK_TRACE_DBG("PROCTH: Susp Pst while STOPPED_INIT\n");
         }
     }
 
@@ -456,7 +520,7 @@ void p9_pgpe_process_wof_ctrl()
                 PkMachineContext  ctx;
                 //Send "Disable Core Stop Updates" IPC to SGPE
                 G_sgpe_control_updt.fields.return_code = 0x0;
-                G_sge_control_updt.fields.action = CTRL_STOP_UPDT_ENABLE_CORE;
+                G_sgpe_control_updt.fields.action = CTRL_STOP_UPDT_ENABLE_CORE;
                 G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_control_updt;
                 ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
                              IPC_MSGID_PGPE_SGPE_CONTROL_STOP_UPDATES,
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/pk_app_cfg.h
index 8b51888f..3d202261 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/pk_app_cfg.h
@@ -42,7 +42,7 @@
 #define BOOT_TEMP_SET_FULL_OCC_IPC_FUNC 1
 #define DEV_DEBUG     1
 #define PK_TRACE_TIMER_OUTPUT 0
-#define SGPE_IPC_ENABLED 0
+#define SGPE_IPC_ENABLED 1
 #define OVERRIDE_OTHER_ENGINES_IRQS 0
 #define OVERRIDE_PSAFE_PSTATE 1
 
@@ -83,7 +83,7 @@
     #define PK_TRACE_ENABLE        1
     #define PK_TRACE_DBG_SUPPRESS  0
     #define PK_TRACE_CRIT_ENABLE   1
-    #define PK_TRACE_CKPT_ENABLE   1
+    #define PK_TRACE_CKPT_ENABLE   0
     #define PK_KERNEL_TRACE_ENABLE 0
 #else                    /*All TRACEs*/
     #define PK_TRACE_ENABLE        1