PM: Fixes for Livelock Scenarios

    -Fixes DPLL Ownership issues during Pstate Start
    -Fixes WOF Enablement and Quad/Core Active Update(STOP11/5) livelock
    scenario
    -Fixes PM Complex Suspend and Quad/Core Active Update(STOP11/5)
    livelock scenario
    -Fixes VDM Droop Suspend STOP entries livelock scenario

    Key_Cronus_Test=PM_REGRESS

Change-Id: I14a0dece4c74bc04618f7d1f3838dbe273bace94
CQ: SW425778
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/57191
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Cronus HW CI <cronushw-ci+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Reviewed-by: YUE DU <daviddu@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

21 files changed, 394 insertions, 265 deletions

diff --git a/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h b/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
index 570fb411..a171b47d 100644
--- a/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
+++ b/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
@@ -111,6 +111,8 @@ enum MESSAGE_ID_DB3
     MSGID_DB3_ENTER_SAFE_MODE           = 0x02,
     MSGID_DB3_REPLAY_DB0                = 0x03,
     MSGID_DB3_DISABLE_SGPE_HANDOFF      = 0x04,
+    MSGID_DB3_SUSPEND_STOP_ENTRY        = 0x05,
+    MSGID_DB3_UNSUSPEND_STOP_ENTRY      = 0x06,
     MSGID_DB3_IMMEDIATE_HALT            = 0xF1,
     MSGID_DB3_RESTORE_STATE_AND_HALT    = 0xF2,
 };
@@ -121,7 +123,9 @@ enum MESSAGEID_PCB_TYPE4_ACK_TYPES
     MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK         = 1,
     MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED         = 2,
     MSGID_PCB_TYPE4_QUAD_MGR_AVAILABLE           = 3,
-    MSGID_PCB_TYPE4_NACK_DROOP_PRESENT           = 4
+    MSGID_PCB_TYPE4_NACK_DROOP_PRESENT           = 4,
+    MSGID_PCB_TYPE4_SUSPEND_ENTRY_ACK            = 5,
+    MSGID_PCB_TYPE4_UNSUSPEND_ENTRY_ACK          = 6
 };
 
 enum DB0_CLIP_BCAST_FIELDS
diff --git a/import/chips/p9/common/pmlib/include/wof_sgpe_pgpe_api.h b/import/chips/p9/common/pmlib/include/wof_sgpe_pgpe_api.h
index f018bd98..e8f843a6 100644
--- a/import/chips/p9/common/pmlib/include/wof_sgpe_pgpe_api.h
+++ b/import/chips/p9/common/pmlib/include/wof_sgpe_pgpe_api.h
@@ -48,14 +48,7 @@ enum MESSAGE_ID_IPI3HI_PGPE_SGPE
 
 enum CTRL_STOP_UPDATES_ACTIONS
 {
-    CTRL_STOP_UPDT_RESERVED0               = 0x0,
     CTRL_STOP_UPDT_ENABLE_CORE             = 0x1,
-    CTRL_STOP_UPDT_ENABLE_QUAD             = 0x2,
-    CTRL_STOP_UPDT_ENABLE_CORE_QUAD        = 0x3,
-    CTRL_STOP_UPDT_RESERVED1               = 0x4,
-    CTRL_STOP_UPDT_DISABLE_CORE            = 0x5,
-    CTRL_STOP_UPDT_DISABLE_QUAD            = 0x6,
-    CTRL_STOP_UPDT_DISABLE_CORE_QUAD       = 0x7
 };
 
 
@@ -65,6 +58,8 @@ enum UPDATE_ACTIVE_TYPES
     UPDATE_ACTIVE_CORES_TYPE_EXIT               = 0x1,
     UPDATE_ACTIVE_QUADS_TYPE_ENTRY              = 0x0,
     UPDATE_ACTIVE_QUADS_TYPE_EXIT               = 0x1,
+    UPDATE_ACTIVE_QUADS_EXIT_TYPE_DONE          = 0x0,
+    UPDATE_ACTIVE_QUADS_EXIT_TYPE_NOTIFY        = 0x1,
     UPDATE_ACTIVE_QUADS_ENTRY_TYPE_DONE         = 0x0,
     UPDATE_ACTIVE_QUADS_ENTRY_TYPE_NOTIFY       = 0x1
 };
@@ -85,6 +80,7 @@ enum SUSPEND_STOP_COMMANDS
 #define SGPE_PGPE_IPC_RC_SUCCESS                0x01
 #define SGPE_PGPE_RC_REQ_WHILE_PENDING_ACK      0x10
 #define SGPE_PGPE_RC_PM_COMPLEX_SUSPEND         0x11
+#define SGPE_PGPE_RC_WOF_DISABLED               0x12
 
 enum IPC_SGPE_PGPE_RETURN_CODES
 {
@@ -116,7 +112,8 @@ typedef union
         uint64_t msg_num                : 4;
         uint64_t update_type            : 1;
         uint64_t entry_type             : 1;
-        uint64_t reserved               : 2;
+        uint64_t exit_type              : 1;
+        uint64_t reserved               : 1;
         uint64_t return_code            : 8;
         uint64_t requested_quads        : 6;
         uint64_t reserved0              : 2;
diff --git a/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h b/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
index 40812374..0a485a72 100644
--- a/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
+++ b/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
@@ -124,6 +124,7 @@ enum PM_CME_FLAGS_DEFS
     CME_FLAGS_STOP_BLOCK_ENTRY_C1                       = 11,
     CME_FLAGS_CORE_QUIESCE_ACTIVE                       = 12,
     CME_FLAGS_PM_DEBUG_HALT_ENABLE                      = 13,
+    CME_FLAGS_DROOP_SUSPEND_ENTRY                       = 14,
     CME_FLAGS_SAFE_MODE                                 = 16,
     CME_FLAGS_PSTATES_SUSPENDED                         = 17,
     CME_FLAGS_SPWU_CHECK_ENABLE                         = 22,
diff --git a/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h
index 521a85fb..99b0d852 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h
@@ -38,7 +38,7 @@
 #define PPE_DEBUG_PTRS_SIZE      CME_DEBUG_PTRS_SIZE
 
 #define PPE_DUMP_PTR_PSTATE_SIZE 0x50
-#define PPE_DUMP_PTR_STOP_SIZE   0x34
+#define PPE_DUMP_PTR_STOP_SIZE   0x3C
 #define PPE_DUMP_PTR_COMMON_SIZE 0xC
 
 #endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
index 0aa0651d..501e56e7 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
@@ -39,7 +39,7 @@ CmePstateRecord G_cme_pstate_record __attribute__((section (".dump_ptr_pstate"))
 
 // CME Stop Header and Structure
 #include "p9_cme_stop.h"
-CmeStopRecord G_cme_stop_record __attribute__((section (".dump_ptr_stop"))) = {{0}, {0}, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, {0}};
+CmeStopRecord G_cme_stop_record __attribute__((section (".dump_ptr_stop"))) = {{0}, {0}, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, {0}};
 
 #if !DISABLE_PERIODIC_CORE_QUIESCE && (NIMBUS_DD_LEVEL == 20 || NIMBUS_DD_LEVEL == 21 || CUMULUS_DD_LEVEL == 10)
 CmeFitRecord G_cme_fit_record = {0, 0, 0, 0, 0xFFFFFFFF, 0};
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
index 97323138..a55194ee 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
@@ -39,7 +39,7 @@
 #include "cmehw_interrupts.h"
 
 #include "p9_cme_irq.h"
-#include "p9_cme_pstate.h"
+#include "p9_cme_stop.h"
 #include "pstate_pgpe_cme_api.h"
 #include "ppe42_cache.h"
 #include "p9_hcode_image_defines.H"
@@ -51,6 +51,7 @@
 //External Globals and globals
 //
 extern CmeRecord G_cme_record;
+extern CmeStopRecord G_cme_stop_record;
 extern CmePstateRecord G_cme_pstate_record;
 extern cmeHeader_t* G_cmeHeader;
 extern LocalPstateParmBlock* G_lppb;
@@ -256,6 +257,43 @@ void p9_cme_pstate_db3_handler(void)
             intercme_direct(INTERCME_DIRECT_IN2, INTERCME_DIRECT_ACK, 0);
         }
     }
+    else if (db3.fields.cme_message_numbern == MSGID_DB3_SUSPEND_STOP_ENTRY)
+    {
+        G_cme_stop_record.core_vdm_droop = CME_MASK_BC;
+
+        if (!(G_cme_stop_record.entry_ongoing ||
+              G_cme_stop_record.exit_ongoing))
+        {
+            p9_cme_stop_eval_eimr_override();
+        }
+
+        //Note: we don't ack back to PGPE. Instead, the STOP code will set the
+        //CME_FLAGS[SUSPEND_ENTRY] whenever it finishes any currently ongoing entry
+
+    }
+    else if (db3.fields.cme_message_numbern == MSGID_DB3_UNSUSPEND_STOP_ENTRY)
+    {
+        G_cme_stop_record.core_vdm_droop = 0;
+        p9_cme_stop_eval_eimr_override();
+
+        //Notify and Receive ack from sibling CME. This syncs up
+        //Quad Manager and Sibling before Quad Manager acks back to
+        //PGPE
+        if (G_cme_pstate_record.qmFlag)
+        {
+            p9_cme_pstate_notify_sib(INTERCME_DIRECT_IN2);
+            send_ack_to_pgpe(MSGID_PCB_TYPE4_UNSUSPEND_ENTRY_ACK);
+        }
+        else
+        {
+            //Wait to receive a notify from Quad Manager
+            //and then ACK back to quad manager
+            while(!(in32_sh(CME_LCL_EISR) & BIT64SH(39)));
+
+            intercme_direct(INTERCME_DIRECT_IN2, INTERCME_DIRECT_ACK, 0);
+        }
+
+    }
     else
     {
         //\todo  Will be done as part of 41947
@@ -265,6 +303,7 @@ void p9_cme_pstate_db3_handler(void)
     PK_TRACE_INF("DB3 Handler Done ");
 }
 
+
 //
 void p9_cme_pstate_init()
 {
@@ -433,8 +472,8 @@ void p9_cme_pstate_init()
 
     if (cme_flags & BIT32(CME_FLAGS_CORE0_GOOD))
     {
-        eimr_clr |=  BIT64(11); //Enable DB3_0
-        eimr_or  |=  BIT64(10); //Disable DB3_1
+        eimr_clr |=  BIT64(10); //Enable DB3_0
+        eimr_or  |=  BIT64(11); //Disable DB3_1
     }
     else
     {
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
index 48f922ab..dbd9efdf 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
@@ -271,9 +271,9 @@ typedef struct
     // need to be a global state for aborting entry
     uint8_t       act_level[MAX_CORES_PER_CME];
     // uint8_t above is processed by stb/lbz in asm, no additional shifting
-
-    // mark the start and the end of entry process
-    uint32_t      entry_pending;
+    // mark the start and the end of entry/exit process
+    uint32_t      entry_ongoing;
+    uint32_t      exit_ongoing;
     // whether core is in running state,
     // used for aborted entry detection or filter wakeup core select in scom address
     uint32_t      core_running;
@@ -289,6 +289,8 @@ typedef struct
     uint32_t      core_suspendwu;
     // core in suspend entry mode, can be used as core select in scom address or data
     uint32_t      core_suspendey;
+    // core needs specifical entry masks due to vdm prolonged droop events
+    uint32_t      core_vdm_droop;
     // core in special wakeup, can be used as core select in scom address or data
     uint32_t      core_in_spwu;
     // core in error state, prevent it being further processed
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
index 6dc58b74..cb42cf86 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
@@ -69,6 +69,7 @@ p9_cme_stop_init()
     G_cme_stop_record.core_blockey = 0;
     G_cme_stop_record.core_suspendwu = G_cme_stop_record.core_blockpc;
     G_cme_stop_record.core_suspendey = 0;
+    G_cme_stop_record.core_vdm_droop = 0;
 
     if (in32(CME_LCL_FLAGS) & BIT32(CME_FLAGS_BLOCK_ENTRY_STOP11))
     {
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
index cf8bd2f4..09ba46b9 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
@@ -87,14 +87,16 @@ p9_cme_stop_pcwu_handler(void)
 
         out32(CME_LCL_EIMR_OR, BITS32(12, 10));
         g_eimr_override |= BITS64(12, 10);
+        G_cme_stop_record.exit_ongoing = 1;
         wrteei(1);
 
         // The actual exit sequence
         p9_cme_stop_exit();
+        G_cme_stop_record.exit_ongoing = 0;
     }
 
     // in case abort, complete pending entry first
-    if (!G_cme_stop_record.entry_pending)
+    if (!G_cme_stop_record.entry_ongoing)
     {
         // re-evaluate stop entry & exit enables
         p9_cme_stop_eval_eimr_override();
@@ -199,14 +201,16 @@ p9_cme_stop_spwu_handler(void)
 
         out32(CME_LCL_EIMR_OR, BITS32(12, 10));
         g_eimr_override |= BITS64(12, 10);
+        G_cme_stop_record.exit_ongoing = 1;
         wrteei(1);
 
         // The actual exit sequence
         p9_cme_stop_exit();
+        G_cme_stop_record.exit_ongoing = 0;
     }
 
     // in case abort, complete pending entry first
-    if (!G_cme_stop_record.entry_pending)
+    if (!G_cme_stop_record.entry_ongoing)
     {
         // re-evaluate stop entry & exit enables
         p9_cme_stop_eval_eimr_override();
@@ -223,13 +227,16 @@ p9_cme_stop_rgwu_handler(void)
 
     out32(CME_LCL_EIMR_OR, BITS32(12, 10));
     g_eimr_override |= BITS64(12, 10);
+    G_cme_stop_record.exit_ongoing = 1;
     wrteei(1);
 
     // The actual exit sequence
     p9_cme_stop_exit();
 
+    G_cme_stop_record.exit_ongoing = 0;
+
     // in case abort, complete pending entry first
-    if (!G_cme_stop_record.entry_pending)
+    if (!G_cme_stop_record.entry_ongoing)
     {
         // re-evaluate stop entry & exit enables
         p9_cme_stop_eval_eimr_override();
@@ -247,14 +254,14 @@ p9_cme_stop_enter_handler(void)
     // Abort Protection
     out32(CME_LCL_EIMR_OR, BITS32(12, 10));
     g_eimr_override |= BITS64(12, 10);
-    G_cme_stop_record.entry_pending = 1;
+    G_cme_stop_record.entry_ongoing = 1;
     wrteei(1);
 
     // The actual entry sequence
     p9_cme_stop_entry();
 
     // Restore Abort Protection
-    G_cme_stop_record.entry_pending = 0;
+    G_cme_stop_record.entry_ongoing = 0;
 
     // re-evaluate stop entry & exit enables
     p9_cme_stop_eval_eimr_override();
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_threads.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_threads.c
index 119791ae..55f7db99 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_threads.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_threads.c
@@ -90,9 +90,19 @@ p9_cme_stop_eval_eimr_override()
            G_cme_stop_record.core_errored    |
            G_cme_stop_record.core_in_spwu    |
            G_cme_stop_record.core_blockey    |
+           G_cme_stop_record.core_vdm_droop  |
            G_cme_stop_record.core_suspendey) & CME_MASK_BC) << SHIFT32(21));
 
     g_eimr_override |= mask_irqs.value;
+
+    if (G_cme_stop_record.core_vdm_droop)
+    {
+        out32(CME_LCL_FLAGS_OR, BIT32(CME_FLAGS_DROOP_SUSPEND_ENTRY));
+    }
+    else
+    {
+        out32(CME_LCL_FLAGS_CLR, BIT32(CME_FLAGS_DROOP_SUSPEND_ENTRY));
+    }
 }
 
 #if !DISABLE_PERIODIC_CORE_QUIESCE && (NIMBUS_DD_LEVEL == 20 || NIMBUS_DD_LEVEL == 21 || CUMULUS_DD_LEVEL == 10)
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
index 4c806450..12775772 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_irq_handlers.c
@@ -102,8 +102,8 @@ void p9_pgpe_irq_handler_occ_sgpe_cme_pvref_error(void* arg, PkIrqId irq)
 {
     PkMachineContext  ctx;
 
-    PK_TRACE_INF("Error IRQ Detected");
     uint64_t oisr = ((uint64_t)(in32(OCB_OISR0)) << 32) | in32(OCB_OISR1);
+    PK_TRACE_INF("Error IRQ Detected OISR=0x%08x%08x", oisr >> 32, oisr);
 
     //OCC Error
     if(oisr & BIT64(2))
@@ -187,6 +187,8 @@ void p9_pgpe_irq_handler_sgpe_err()
 {
     PK_TRACE_INF("SGPE Error");
 
+    g_oimr_override |=   BIT64(8);
+    out32(OCB_OIMR0_OR,  BIT32(8));
     out32(OCB_OISR0_CLR, BIT32(8));
 
     //Optrace
@@ -262,6 +264,8 @@ void p9_pgpe_irq_handler_system_xstop(void* arg, PkIrqId irq)
     PK_TRACE_INF("SYSTEM XSTOP");
     PkMachineContext  ctx;
 
+    g_oimr_override |=   BIT64(15);
+    out32(OCB_OIMR0_OR,  BIT32(15));
     out32(OCB_OISR0_CLR, BIT32(15));
 
     //Optrace
@@ -454,10 +458,11 @@ void p9_pgpe_irq_handler_cme_err()
 
         if (opit5prQuad)
         {
-            PK_TRACE_DBG("CER:Quad[%d]", q);
-
             G_pgpe_pstate_record.errorQuads |= QUAD_MASK(q);
 
+            PK_TRACE_INF("CER:Quad[%d]", q);
+
+
             //1.1 Halt both CMEs in the quad containing faulted CME,
             if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
             {
@@ -476,7 +481,6 @@ void p9_pgpe_irq_handler_cme_err()
                 }
             }
 
-            PK_TRACE_DBG("CER:Quad[%d] CMEs Halted", q);
 
             //1.2 The quad in error is stepped out of resonance by the PGPE. This keeps the cores that may be
             //  running in the quad operating. There is a momentary rise in power as resonance is disabled.
@@ -522,7 +526,6 @@ void p9_pgpe_irq_handler_cme_err()
                     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QACCR, q), value);
                 }
 
-                PK_TRACE_DBG("CER:Quad[%d] ResClk Disabled", q);
             }
 
             //1.3 Move DPLL to Fsafe (~100us)
@@ -552,12 +555,11 @@ void p9_pgpe_irq_handler_cme_err()
             }
             while (!freq_done);
 
-            PK_TRACE_DBG("CER:Quad[%d] DPLL Moved to Fsafe", q);
 
             //2. The quad in error is removed from the expected Ack vector.
             G_pgpe_pstate_record.activeQuads &= (~QUAD_MASK(q));
             G_pgpe_pstate_record.activeDB &= ~(QUAD_ALL_CORES_MASK(q));
-            PK_TRACE_DBG("CER: Quad[%d] Removed from activeQuads", q);
+            PK_TRACE_DBG("CER: Quad[%d] ResclkDisabled, Moved to Fsafe, and removed from activeQuads", q);
         }
     }
 
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
index 39e1da64..50953b25 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_main.c
@@ -140,7 +140,7 @@ IRQ_HANDLER_DEFAULT            //OCCHW_IRQ_RESERVED_63
 EXTERNAL_IRQ_TABLE_END
 
 #define  KERNEL_STACK_SIZE  512
-#define  THREAD_PROCESS_STACK_SIZE  512
+#define  THREAD_PROCESS_STACK_SIZE  768
 #define  THREAD_ACTUATE_STACK_SIZE  768
 
 #define  PGPE_THREAD_PRIORITY_PROCESS_REQUESTS  1
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
index 69219866..234be95b 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
@@ -65,8 +65,8 @@ GPE_BUFFER(ipcmsg_p2s_suspend_stop_t G_sgpe_suspend_stop);
 //Local Functions
 void p9_pgpe_handle_nacks(uint32_t origCoreVector, uint32_t origAckVector, uint32_t expectedAcks);
 void p9_pgpe_pstate_freq_updt();
-void p9_pgpe_droop_throttle();
-void p9_pgpe_droop_unthrottle();
+inline void p9_pgpe_droop_throttle() __attribute__((always_inline));
+inline void p9_pgpe_droop_unthrottle() __attribute__((always_inline));
 
 //
 //p9_pgpe_pstate_init
@@ -130,6 +130,7 @@ void p9_pgpe_pstate_init()
     G_pgpe_pstate_record.pQuadState1->fields.quad5_pstate = 0xff;
     G_pgpe_pstate_record.pQuadState1->fields.active_cores = 0x0;
     G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads = 0x0;
+    G_pgpe_pstate_record.activeCoreUpdtAction = ACTIVE_CORE_UPDATE_ACTION_ERROR;
 
     //Create Semaphores
     pk_semaphore_create(&(G_pgpe_pstate_record.sem_actuate), 0, 1);
@@ -261,7 +262,7 @@ void p9_pgpe_pstate_do_auction()
 //
 void p9_pgpe_pstate_apply_clips()
 {
-    PK_TRACE_INF("APC: Applying Clips");
+    PK_TRACE_DBG("APC: Applying Clips");
     uint32_t q;
 
     for (q = 0; q < MAX_QUADS; q++)
@@ -555,6 +556,8 @@ void p9_pgpe_wait_cme_db_ack(uint32_t quadAckExpect, uint32_t expectedAck)
 
                         case MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK: //0x1
                         case MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED: //0x2
+                        case MSGID_PCB_TYPE4_SUSPEND_ENTRY_ACK:    //0x5
+                        case MSGID_PCB_TYPE4_UNSUSPEND_ENTRY_ACK:  //0x6
 
                             //Check if ack is same as expected
                             if (ack != expectedAck)
@@ -778,26 +781,17 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
     //Move voltage only if raising it. Otherwise, we lower it later after
     //sending Pstate Start DB0. This is to make sure VDMs are not affected in
     //this window
-    if (G_pgpe_pstate_record.eVidCurr >= G_pgpe_pstate_record.eVidNext)
-    {
-        for (q = 0; q < MAX_QUADS; q++)
-        {
-            if (!(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q)))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, q), dpllFreq.value);//Write DPLL
-            }
-        }
-    }
-    else if(G_pgpe_pstate_record.eVidCurr < G_pgpe_pstate_record.eVidNext)
+    if(G_pgpe_pstate_record.eVidCurr < G_pgpe_pstate_record.eVidNext)
     {
         p9_pgpe_pstate_updt_ext_volt(G_pgpe_pstate_record.eVidNext); //update voltage
+    }
 
-        for (q = 0; q < MAX_QUADS; q++)
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if (!(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q)))
         {
-            if (!(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q)))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, q), dpllFreq.value);//Write DPLL
-            }
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26));
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, q), dpllFreq.value);//Write DPLL
         }
     }
 
@@ -1025,6 +1019,20 @@ void p9_pgpe_pstate_stop()
     db0_stop.value = 0;
     db0_stop.fields.msg_id = MSGID_DB0_STOP_PSTATE_BROADCAST;
 
+    //Send PSTATE_STOP DB0
+    p.db0val             = db0_stop.value;
+    p.type               = PGPE_DB0_TYPE_UNICAST;
+    p.targetCores        = G_pgpe_pstate_record.activeDB;
+    p.waitForAcks        = PGPE_DB_ACK_WAIT_CME;
+    p.expectedAckFrom    = G_pgpe_pstate_record.activeQuads;
+    p.expectedAckValue   = MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED;
+    p9_pgpe_send_db0(p);
+
+    //Note: We take away DPLL control from CME after stopping pstates on it
+    //Otherwise, we can have a case where PGPE Heartbeat loss occurs after
+    //PGPE has taken away DPLL control from CME, but haven't sent Pstate Stop
+    //DB0. CME unmasks Heartbeat Loss interrupt upon receiving Pstate Start,
+    //and masks it upon receiving Pstate Stop.
     for (q = 0; q < MAX_QUADS; q++)
     {
         if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
@@ -1047,14 +1055,6 @@ void p9_pgpe_pstate_stop()
         }
     }
 
-    //Send PSTATE_STOP DB0
-    p.db0val             = db0_stop.value;
-    p.type               = PGPE_DB0_TYPE_UNICAST;
-    p.targetCores        = G_pgpe_pstate_record.activeDB;
-    p.waitForAcks        = PGPE_DB_ACK_WAIT_CME;
-    p.expectedAckFrom    = G_pgpe_pstate_record.activeQuads;
-    p.expectedAckValue   = MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED;
-    p9_pgpe_send_db0(p);
 
     //Set status in OCC_Scratch2
     uint32_t occScr2 = in32(OCB_OCCS2);
@@ -1126,14 +1126,14 @@ void p9_pgpe_pstate_wof_ctrl(uint32_t action)
         {
             p9_pgpe_pstate_send_ctrl_stop_updt(CTRL_STOP_UPDT_ENABLE_CORE);
             activeCores = G_sgpe_control_updt.fields.active_cores << 8;
-            activeQuads = G_sgpe_control_updt.fields.active_quads << 2;
+            G_pgpe_pstate_record.activeCoreUpdtAction = ACTIVE_CORE_UPDATE_ACTION_PROCESS_AND_ACK;
         }
         else
         {
             activeCores = G_pgpe_pstate_record.activeDB;
-            activeQuads = G_pgpe_pstate_record.activeQuads;
         }
 
+        activeQuads = G_pgpe_pstate_record.activeQuads;
         G_pgpe_pstate_record.wofStatus = WOF_ENABLED;
 
         //Set to value returned by SGPE or initial value determined during boot(equal to configured cores)
@@ -1162,11 +1162,12 @@ void p9_pgpe_pstate_wof_ctrl(uint32_t action)
     }
     else if (action == PGPE_ACTION_WOF_OFF)
     {
-        //In WOF Phase >= 2, we ask SGPE to stop sending active core updates
+        //In WOF Phase >= 2, we take a note that WOF has been disabled, and
+        //simply ACK any active cores updates that come from SGPE.
         if ((G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_ENABLE_VRATIO) ||
             (G_pgpe_header_data->g_pgpe_qm_flags & PGPE_FLAG_VRATIO_MODIFIER))
         {
-            p9_pgpe_pstate_send_ctrl_stop_updt(CTRL_STOP_UPDT_DISABLE_CORE);
+            G_pgpe_pstate_record.activeCoreUpdtAction = ACTIVE_CORE_UPDATE_ACTION_ACK_ONLY;
         }
 
         G_pgpe_pstate_record.wofStatus = WOF_DISABLED;
@@ -1181,14 +1182,16 @@ void p9_pgpe_pstate_wof_ctrl(uint32_t action)
 //
 void p9_pgpe_pstate_process_quad_entry_notify(uint32_t quadsRequested)
 {
-    uint32_t q;
+    uint32_t q, c;
     qppm_dpll_freq_t dpllFreq;
     dpllFreq.value = 0;
-    ocb_qcsr_t qcsr;
-    qcsr.value = in32(OCB_QCSR);
-    uint64_t value;
+    db0_parms_t p;
+    pgpe_db0_stop_ps_bcast_t db0_stop;
+    uint32_t target_cores;
 
     G_pgpe_pstate_record.activeQuads &= ~quadsRequested;
+    db0_stop.value = 0;
+    db0_stop.fields.msg_id = MSGID_DB0_STOP_PSTATE_BROADCAST;
 
     PK_TRACE_INF("QE:(Notify) QReq=0x%x QAct=%x \n", quadsRequested, G_pgpe_pstate_record.activeQuads);
 
@@ -1196,34 +1199,35 @@ void p9_pgpe_pstate_process_quad_entry_notify(uint32_t quadsRequested)
     {
         if (quadsRequested & QUAD_MASK(q))
         {
-            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26)); //Open DPLL for SCOMs
+            target_cores = 0;
 
-            G_pgpe_pstate_record.activeDB &= ~(QUAD_ALL_CORES_MASK(q));
-            out32(OCB_OPIT4PRA_CLR, QUAD_ALL_CORES_MASK(q)); //Clear any pending PCB_Type4
-
-            //CME_Scratch0[DB0_PROCESSING_ENABLE]=0
-            if (qcsr.fields.ex_config &  (0x800 >> (q * 2)))
+            if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
             {
-                GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
-                value &= ~BIT64(CME_SCRATCH_DB0_PROCESSING_ENABLE);
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 0), value);
-            }
+                for (c = FIRST_CORE_FROM_QUAD(q); c < LAST_CORE_FROM_QUAD(q); c++)
+                {
+                    if (G_pgpe_pstate_record.activeDB & CORE_MASK(c))
+                    {
+                        target_cores |= CORE_MASK(c);
+                    }
+                }
 
-            if (qcsr.fields.ex_config &  (0x400 >> (q * 2)))
-            {
-                GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 1), value);
-                value &= ~BIT64(CME_SCRATCH_DB0_PROCESSING_ENABLE);
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_SRTCH0, q, 1), value);
-            }
 
+                //Send PSTATE_STOP DB0
+                p.db0val             = db0_stop.value;
+                p.type               = PGPE_DB0_TYPE_UNICAST;
+                p.targetCores        = target_cores;
+                p.waitForAcks        = PGPE_DB_ACK_WAIT_CME;
+                p.expectedAckFrom    = QUAD_MASK(q);
+                p.expectedAckValue   = MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED;
+                p9_pgpe_send_db0(p);
 
-            if (G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
-            {
                 G_pgpe_pstate_record.psTarget.fields.quads[q] = 0xFF;
                 G_pgpe_pstate_record.psCurr.fields.quads[q] = 0xFF;
                 G_pgpe_pstate_record.psNext.fields.quads[q] = 0xFF;
                 G_pgpe_pstate_record.psComputed.fields.quads[q] = 0xFF;
 
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26)); //Open DPLL for SCOMs
+
                 //Write "Safe Frequency" for quad about to enter STOP
                 //Note, we set fmax = fmult = fmin
                 dpllFreq.fields.fmax  = G_gppb->dpll_pstate0_value - G_pgpe_pstate_record.safePstate;
@@ -1231,11 +1235,20 @@ void p9_pgpe_pstate_process_quad_entry_notify(uint32_t quadsRequested)
                 dpllFreq.fields.fmin  = dpllFreq.fields.fmax;
                 GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_FREQ, q), dpllFreq.value);
             }
+            else
+            {
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26)); //Open DPLL for SCOMs
+            }
+
+            G_pgpe_pstate_record.activeDB &= ~(QUAD_ALL_CORES_MASK(q));
+            out32(OCB_OPIT4PRA_CLR, QUAD_ALL_CORES_MASK(q)); //Clear any pending PCB_Type4
+
         }
     }
 
     p9_pgpe_pstate_updt_actual_quad();
 
+    G_pgpe_pstate_record.pendingActiveQuadUpdtDone = 1;
 }
 
 //
@@ -1253,12 +1266,13 @@ void p9_pgpe_pstate_process_quad_entry_done(uint32_t quadsRequested)
         GPE_PUTSCOM(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
     }
 
+    G_pgpe_pstate_record.pendingActiveQuadUpdtDone = 0;
 }
 
 //
 //p9_pgpe_pstate_process_quad_exit
 //
-void p9_pgpe_pstate_process_quad_exit(uint32_t quadsRequested)
+void p9_pgpe_pstate_process_quad_exit_notify(uint32_t quadsRequested)
 {
     uint32_t q;
     qppm_dpll_freq_t dpllFreq;
@@ -1301,10 +1315,19 @@ void p9_pgpe_pstate_process_quad_exit(uint32_t quadsRequested)
     //registration msg from the quad is received.
     G_pgpe_pstate_record.pendQuadsRegisterReceive |= quadsRequested;
 
-    PK_TRACE_INF("QX: (Done), QReq=0x%x,QAct=%x\n", quadsRequested, G_pgpe_pstate_record.activeQuads);
+    PK_TRACE_INF("QX: (Notify), QReq=0x%x,QAct=%x\n", quadsRequested, G_pgpe_pstate_record.activeQuads);
+
+    G_pgpe_pstate_record.pendingActiveQuadUpdtDone = 1;
 }
 
 
+void p9_pgpe_pstate_process_quad_exit_done()
+{
+
+    G_pgpe_pstate_record.pendingActiveQuadUpdtDone = 0;
+    PK_TRACE_INF("QX: (Done) QAct=%x\n", G_pgpe_pstate_record.activeQuads);
+}
+
 //
 //p9_pgpe_pstate_send_ctrl_stop_updt
 //
@@ -1351,13 +1374,13 @@ void p9_pgpe_pstate_send_ctrl_stop_updt(uint32_t action)
 //
 //p9_pgpe_pstate_send_suspend_stop
 //
-void p9_pgpe_pstate_send_suspend_stop(uint32_t command)
+void p9_pgpe_pstate_send_suspend_stop()
 {
     p9_pgpe_optrace(PRC_PM_SUSP);
 
     int rc;
     G_sgpe_suspend_stop.fields.msg_num = MSGID_PGPE_SGPE_SUSPEND_STOP;
-    G_sgpe_suspend_stop.fields.command  = command;
+    G_sgpe_suspend_stop.fields.command  = SUSPEND_STOP_SUSPEND_ENTRY_EXIT;
     G_sgpe_suspend_stop.fields.return_code = 0x0;
     G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_suspend_stop;
     ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
@@ -1368,35 +1391,11 @@ void p9_pgpe_pstate_send_suspend_stop(uint32_t command)
     //send the command
     rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
 
-    PK_TRACE_INF("SUSP:Sent Suspend Stop(cmd=0x%x)", command);
-
     if(rc)
     {
         PK_TRACE_ERR("SUSP:Suspend Stop IPC FAIL");
         PGPE_TRACE_AND_PANIC(PGPE_SGPE_IPC_SEND_FAIL);
     }
-
-    //Just block until SGPE writes the return code field
-    //It is assumed that SGPE will write this field right before
-    //calling ipc_send_rsp(). Also, note that PGPE will get the ACK
-    //in the form of IPC interrupt which will call the p9_pgpe_process_ack_sgpe_suspend_stop
-    //
-    // Alternative is to wait until ACKs arrives and do processing
-    //in the IPC ACK handler, but this requires opening up IPC interrupt. Instead, we
-    //do processing after blocking here
-    while ((G_sgpe_suspend_stop.fields.return_code == IPC_SGPE_PGPE_RC_NULL) &&
-           !G_pgpe_pstate_record.severeFault[SAFE_MODE_FAULT_SGPE])
-    {
-        dcbi(((void*)(&G_sgpe_suspend_stop)));
-    }
-
-    if (G_sgpe_suspend_stop.fields.return_code != IPC_SGPE_PGPE_RC_SUCCESS)
-    {
-        PK_TRACE_ERR("ERROR: SGPE Suspend STOP Bad RC. Halting PGPE!");
-        PGPE_TRACE_AND_PANIC(PGPE_SGPE_SUSPEND_STOP_BAD_ACK);
-    }
-
-    PK_TRACE_INF("SUSP:Suspend Stop(cmd=0x%x) ACKed", command);
 }
 
 //
@@ -1497,7 +1496,7 @@ void p9_pgpe_pstate_safe_mode()
     {
         async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].cmd;
         ipcmsg_s2p_update_active_quads_t* args = (ipcmsg_s2p_update_active_quads_t*)async_cmd->cmd_data;
-        p9_pgpe_pstate_process_quad_exit(args->fields.requested_quads << 2);
+        p9_pgpe_pstate_process_quad_exit_notify(args->fields.requested_quads << 2);
 
         //activeQuads isn't updated until registration, so we OR with requested quads.
         args->fields.return_active_quads = G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads >> 2;
@@ -1510,6 +1509,7 @@ void p9_pgpe_pstate_safe_mode()
     {
         async_cmd = (ipc_async_cmd_t*)G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd;
         ipcmsg_s2p_update_active_cores_t* args = (ipcmsg_s2p_update_active_cores_t*)async_cmd->cmd_data;
+        G_pgpe_pstate_record.activeCores &= ~(args->fields.active_cores << 8);
         args->fields.return_active_cores = G_pgpe_pstate_record.activeCores >> 8;
         args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
@@ -1561,41 +1561,8 @@ void p9_pgpe_pstate_safe_mode()
 //
 void p9_pgpe_pstate_pm_complex_suspend()
 {
-    int q = 0;
-    ocb_qcsr_t qcsr;
-
-    //Send IPC and wait for ACK
-    p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_SUSPEND_ENTRY_EXIT);
-
-    //Change PMCR ownership
-    PK_TRACE_INF("SUSP: Setting SCOM Ownership of PMCRs");
-    qcsr.value = in32(OCB_QCSR);
-
-    //Set LMCR for each CME
-    for (q = 0; q < MAX_QUADS; q++)
-    {
-        if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
-        {
-            //CME0 within this quad
-            if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 0), BIT64(0));
-            }
-
-            //CME1 within this quad
-            if (qcsr.fields.ex_config & QUAD_EX1_MASK(q))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 1), BIT64(0));
-            }
-        }
-    }
-
-    //OP Trace and Set OCCS2[PM_COMPLEX_SUSPENDED)
-    p9_pgpe_optrace(ACK_PM_SUSP);
-    uint32_t occScr2 = in32(OCB_OCCS2);
-    occScr2 |= BIT32(PM_COMPLEX_SUSPENDED);
-    G_pgpe_pstate_record.pstatesStatus = PSTATE_PM_SUSPENDED;
-    out32(OCB_OCCS2, occScr2);
+    //Send Suspend Stop IPC
+    p9_pgpe_pstate_send_suspend_stop();
 }
 
 
@@ -2059,13 +2026,61 @@ void p9_pgpe_pstate_write_core_throttle(uint32_t throttleData, uint32_t enable_r
 //
 //p9_pgpe_droop_throttle
 //
-void p9_pgpe_droop_throttle()
+inline void p9_pgpe_droop_throttle()
 {
 
     uint32_t q;
-    //1.  PGPE sends IPC to SGPE to Suspend Stop Entries Only, and poll for Success return code (do not open IPCs, ignore the subsequent ACK).  This will prevent a core from going into Stop2 and missing the subsequent "unthrottle."
-    p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_SUSPEND_ENTRY);
+    ocb_qcsr_t qcsr;
+    qcsr.value = in32(OCB_QCSR);
+    uint64_t value;
+    uint32_t ex;
 
+    //1.  PGPE sends Suspend Stop Entry DB3 to all active CMEs
+    db3_parms_t p;
+    p.db3val = (uint64_t)MSGID_DB3_SUSPEND_STOP_ENTRY << 56;
+    p.db0val = 0;
+    p.targetCores       = G_pgpe_pstate_record.activeDB;
+    p.waitForAcks       = PGPE_DB_ACK_SKIP; //We skip ACKs here bc CME will set CME_FLAGS[]
+    p.expectedAckFrom   = G_pgpe_pstate_record.activeQuads;
+    p.expectedAckValue  = MSGID_PCB_TYPE4_SUSPEND_ENTRY_ACK;
+    p.checkNACKs        = PGPE_DB3_SKIP_CHECK_NACKS;
+
+    p9_pgpe_send_db3(p);
+
+    //We poll on CME_FLAGS[] here. The CME doesn't send an ACK for SUSPEND_ENTRY DB3.
+    //Instead, the stop code on CME will set the CME_FLAGS[] when it has suspended stop
+    //entries
+    uint32_t expectedCMEs = 0;
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if (G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
+        {
+            expectedCMEs |= (qcsr.fields.ex_config & (QUAD_EX0_MASK(q) | QUAD_EX1_MASK(q)));
+        }
+    }
+
+    PK_TRACE_INF("DTH: Expected CMEs=0x%x", expectedCMEs);
+
+    while (expectedCMEs != 0)
+    {
+        for (q = 0; q < MAX_QUADS; q++)
+        {
+            for (ex = 0; ex < 2; ex++)
+            {
+                if (expectedCMEs & (QUAD_EX0_MASK(q) >> ex))
+                {
+                    GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_FLAGS, q, ex), value);
+
+                    if (value & BIT64(CME_FLAGS_DROOP_SUSPEND_ENTRY))
+                    {
+                        expectedCMEs &= (~ (QUAD_EX0_MASK(q) >> ex));
+                        PK_TRACE_INF("DTH: Suspend Entry Set Expected CMEs=0x%x", expectedCMEs);
+                    }
+                }
+            }
+        }
+    }
 
     //2.  Call the core_instruction_throttle() procedure to enable throttle (same as used by FIT).
     p9_pgpe_pstate_write_core_throttle(CORE_IFU_THROTTLE, RETRY);
@@ -2096,14 +2111,23 @@ void p9_pgpe_droop_throttle()
 //
 //p9_pgpe_droop_unthrottle
 //
-void p9_pgpe_droop_unthrottle()
+inline void p9_pgpe_droop_unthrottle()
 {
 
     //1.  Call the core_instruction_throttle() procedure to disable throttle (same as used by FIT).
     p9_pgpe_pstate_write_core_throttle(CORE_THROTTLE_OFF, RETRY);
 
-    //2.  PGPE sends IPC to SGPE to Unsuspend STOP entries & poll for Success return code (do not open IPCs, ignore the subsequent ACK).
-    p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_UNSUSPEND_ENTRY);
+    //2.  PGPE sends Unsuspend Stop Entry DB3 to all active CMEs
+    db3_parms_t p;
+    p.db3val = (uint64_t)MSGID_DB3_UNSUSPEND_STOP_ENTRY << 56;
+    p.db0val = 0;
+    p.targetCores       = G_pgpe_pstate_record.activeDB;
+    p.waitForAcks       = PGPE_DB_ACK_WAIT_CME;
+    p.expectedAckFrom   = G_pgpe_pstate_record.activeQuads;
+    p.expectedAckValue  = MSGID_PCB_TYPE4_UNSUSPEND_ENTRY_ACK;
+    p.checkNACKs        = PGPE_DB3_SKIP_CHECK_NACKS;
+
+    p9_pgpe_send_db3(p);
 
     //3.  Send Doorbell0 PMSR Update with message Clear Pstates Suspended to all configured cores in the active Quads.
     p9_pgpe_pstate_send_pmsr_updt(DB0_PMSR_UPDT_CLEAR_PSTATES_SUSPENDED,
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
index 6d1f335d..e05c144e 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.h
@@ -121,6 +121,13 @@ enum OCCLFIR_PGPE
     OCCLFIR_PROLONGED_DROOP_DETECTED = 60
 };
 
+enum ACTIVE_CORE_UPDATE_ACTION
+{
+    ACTIVE_CORE_UPDATE_ACTION_ERROR             = 0x0,
+    ACTIVE_CORE_UPDATE_ACTION_PROCESS_AND_ACK   = 0x1,
+    ACTIVE_CORE_UPDATE_ACTION_ACK_ONLY          = 0x2
+};
+
 //Task list entry
 typedef struct ipc_req
 {
@@ -180,6 +187,8 @@ typedef struct
     uint32_t errorQuads;
     uint32_t updatePGPEBeacon;
     uint8_t severeFault[4];
+    uint32_t pendingActiveQuadUpdtDone;
+    uint32_t activeCoreUpdtAction;
 } PgpePstateRecord __attribute__ ((aligned (8)));
 
 
@@ -235,7 +244,8 @@ void p9_pgpe_pstate_wof_ctrl(uint32_t action);
 //SGPE Comminucation/Processing
 void p9_pgpe_pstate_process_quad_entry_notify(uint32_t quadsAffected);
 void p9_pgpe_pstate_process_quad_entry_done(uint32_t quadsAffected);
-void p9_pgpe_pstate_process_quad_exit(uint32_t quadsAffected);
+void p9_pgpe_pstate_process_quad_exit_notify(uint32_t quadsAffected);
+void p9_pgpe_pstate_process_quad_exit_done();
 void p9_pgpe_pstate_send_suspend_stop();
 void p9_pgpe_pstate_send_ctrl_stop_updt(uint32_t ctrl);
 
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
index aa8fe255..031617ec 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
@@ -243,7 +243,7 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
 
                             if (args_wof_vfrt->active_quads ==  G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads)
                             {
-                                p9_pgpe_pstate_process_quad_exit(args->fields.requested_quads << 2);
+                                p9_pgpe_pstate_process_quad_exit_notify(args->fields.requested_quads << 2);
                                 args->fields.return_active_quads = args_wof_vfrt->active_quads >> 2;
                                 args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
                                 G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_QUADS_UPDT].pending_ack = 0;
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
index 176b1132..c8dccb78 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
@@ -44,16 +44,16 @@ extern GlobalPstateParmBlock* G_gppb;
 //
 //Local Function Prototypes
 //
-void p9_pgpe_process_sgpe_updt_active_cores();
-void p9_pgpe_process_sgpe_updt_active_quads();
-void p9_pgpe_process_start_stop();
-void p9_pgpe_process_clip_updt();
-void p9_pgpe_process_wof_ctrl();
-void p9_pgpe_process_wof_vfrt();
-void p9_pgpe_process_set_pmcr_req();
-void p9_pgpe_process_registration();
-void p9_pgpe_process_ack_sgpe_ctrl_stop_updt();
-void p9_pgpe_process_ack_sgpe_suspend_stop();
+inline void p9_pgpe_process_sgpe_updt_active_cores() __attribute__((always_inline));
+inline void p9_pgpe_process_sgpe_updt_active_quads() __attribute__((always_inline));
+inline void p9_pgpe_process_start_stop() __attribute__((always_inline));
+inline void p9_pgpe_process_clip_updt() __attribute__((always_inline));
+inline void p9_pgpe_process_wof_ctrl() __attribute__((always_inline));
+inline void p9_pgpe_process_wof_vfrt() __attribute__((always_inline));
+inline void p9_pgpe_process_set_pmcr_req() __attribute__((always_inline));
+inline void p9_pgpe_process_registration() __attribute__((always_inline));
+inline void p9_pgpe_process_ack_sgpe_ctrl_stop_updt() __attribute__((always_inline));
+inline void p9_pgpe_process_ack_sgpe_suspend_stop() __attribute__((always_inline));
 
 //
 //Process Request Thread
@@ -128,7 +128,8 @@ void p9_pgpe_thread_process_requests(void* arg)
             p9_pgpe_process_sgpe_updt_active_quads();
         }
 
-        if(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing == 1)
+        if ((G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_PSTATE_START_STOP].pending_processing == 1) &&
+            (G_pgpe_pstate_record.pendingActiveQuadUpdtDone != 1))
         {
             //During process_start_stop, it's pending_ack bit is cleared
             p9_pgpe_process_start_stop();
@@ -203,7 +204,7 @@ void p9_pgpe_thread_process_requests(void* arg)
 //
 //p9_pgpe_process_sgpe_updt_active_cores
 //
-void p9_pgpe_process_sgpe_updt_active_cores()
+inline void p9_pgpe_process_sgpe_updt_active_cores()
 {
     PK_TRACE_DBG("PTH: Core Updt Entry");
     uint32_t c, ack_now = 0, bad_rc = 0;
@@ -215,12 +216,21 @@ void p9_pgpe_process_sgpe_updt_active_cores()
 
     if(G_pgpe_pstate_record.wofStatus != WOF_ENABLED)
     {
-        PK_TRACE_DBG("PTH: C Updt(WOF_Disabled)");
-        args->fields.return_code = PGPE_WOF_RC_NOT_ENABLED;
-        G_pgpe_optrace_data.word[0] = PGPE_OP_CORES_ACTIVE_IN_WOF_DISABLED;
-        p9_pgpe_optrace(UNEXPECTED_ERROR);
         ack_now = 1;
-        bad_rc = 1;
+
+        if (G_pgpe_pstate_record.activeCoreUpdtAction == ACTIVE_CORE_UPDATE_ACTION_ERROR)
+        {
+            G_pgpe_optrace_data.word[0] = PGPE_OP_CORES_ACTIVE_IN_WOF_DISABLED;
+            p9_pgpe_optrace(UNEXPECTED_ERROR);
+            bad_rc = 1;
+            PK_TRACE_DBG("PTH: Core Updt(WOF_Disabled) Error");
+        }
+        else
+        {
+            args->fields.return_active_cores = G_pgpe_pstate_record.activeCores >> 8;
+            ack_now = 1;
+            PK_TRACE_DBG("PTH: Core Updt(WOF_Disabled) Ack Only");
+        }
     }
     else
     {
@@ -284,14 +294,21 @@ void p9_pgpe_process_sgpe_updt_active_cores()
     {
         G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].pending_ack = 0;
         ipc_send_rsp(G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_SGPE_ACTIVE_CORES_UPDT].cmd, IPC_RC_SUCCESS);
-        args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
         PK_TRACE_DBG("PTH: Core Entry ACK back to SGPE");
         p9_pgpe_optrace(ACK_CORES_ACTV);
 
+        //We set the return code after sending the ipc response. This is because
+        //SGPE is polling on the return code and will move on as soon as it is set.
+        //To prevent any race condition, we set the return code afterwards.
         if (bad_rc == 1)
         {
+            args->fields.return_code = SGPE_PGPE_RC_WOF_DISABLED;
             PGPE_OPTIONAL_TRACE_AND_PANIC(PGPE_SGPE_IPC_ACK_BAD_RC);
         }
+        else
+        {
+            args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
+        }
     }
 
     PK_TRACE_DBG("PTH: Core Updt Exit");
@@ -300,9 +317,9 @@ void p9_pgpe_process_sgpe_updt_active_cores()
 //
 //p9_pgpe_process_sgpe_updt_active_quads
 //
-void p9_pgpe_process_sgpe_updt_active_quads()
+inline void p9_pgpe_process_sgpe_updt_active_quads()
 {
-    PK_TRACE_DBG("PTH: Quad Updt Start");
+    PK_TRACE_DBG("QX: Quad Updt Start");
     uint32_t ack_now = 1;
     uint32_t SS;
     uint32_t quads_requested;
@@ -341,28 +358,39 @@ void p9_pgpe_process_sgpe_updt_active_quads()
     //EXIT
     else
     {
-        //Update Shared Memory Region
-        G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (args->fields.requested_quads << 2);
-
-        //WOF Enabled
-        //If WOF_ENABLED=1, and pstatesStatus == ACITVE, then request for WOF_VFRT
-        //Otherwise, we don't as SAFE_MODE or PM_COMPLEX_SUSPEND or STOP is pending
-        if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED && G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
+        if (args->fields.exit_type == UPDATE_ACTIVE_QUADS_ENTRY_TYPE_NOTIFY)
         {
-            PK_TRACE_INF("PTH: OCCLFG[30] set");
-            out32(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
-            ack_now = 0;
+            //Update Shared Memory Region
+            G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads |= (args->fields.requested_quads << 2);
+
+            //WOF Enabled
+            //If WOF_ENABLED=1, and pstatesStatus == ACITVE, then request for WOF_VFRT
+            //Otherwise, we don't as SAFE_MODE or PM_COMPLEX_SUSPEND or STOP is pending
+            if(G_pgpe_pstate_record.wofStatus == WOF_ENABLED && G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
+            {
+                PK_TRACE_INF("PTH: OCCLFG[30] set");
+                out32(OCB_OCCFLG_OR, BIT32(REQUESTED_ACTIVE_QUAD_UPDATE));//Set OCCFLG[REQUESTED_ACTIVE_QUAD_UPDATE]
+                ack_now = 0;
+            }
+            else
+            {
+                p9_pgpe_pstate_process_quad_exit_notify(args->fields.requested_quads << 2);
+                args->fields.return_active_quads = G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads >>
+                                                   2; //activeQuads isn't updated until registration, so we OR with requested quads.
+                args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
+            }
         }
         else
         {
-            p9_pgpe_pstate_process_quad_exit(args->fields.requested_quads << 2);
+            p9_pgpe_pstate_process_quad_exit_done();
             args->fields.return_active_quads = G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads >>
                                                2; //activeQuads isn't updated until registration, so we OR with requested quads.
             args->fields.return_code = IPC_SGPE_PGPE_RC_SUCCESS;
         }
+
     }
 
-    PK_TRACE_INF("PTH: type=0x%x(EN=0,EX=1), req=0x%x,active=0x%x,ret=0x%x", args->fields.update_type,
+    PK_TRACE_INF("QU: type=0x%x(EN=0,EX=1), req=0x%x,active=0x%x,ret=0x%x", args->fields.update_type,
                  args->fields.requested_quads, G_pgpe_pstate_record.activeQuads, args->fields.return_active_quads);
     G_pgpe_optrace_data.word[0] = (args->fields.requested_quads << 26) |
                                   (SS << 24) |
@@ -376,10 +404,10 @@ void p9_pgpe_process_sgpe_updt_active_quads()
         p9_pgpe_optrace(ACK_QUAD_ACTV);
     }
 
-    PK_TRACE_DBG("PTH: Quad Updt End");
+    PK_TRACE_DBG("QX: Quad Updt End");
 }
 
-void p9_pgpe_process_start_stop()
+inline void p9_pgpe_process_start_stop()
 {
     PK_TRACE_DBG("PTH: Start/Stop Entry");
     uint32_t ack_now = 1, bad_rc = 0;
@@ -505,7 +533,7 @@ void p9_pgpe_process_start_stop()
 //
 //p9_pgpe_process_clip_updt
 //
-void p9_pgpe_process_clip_updt()
+inline void p9_pgpe_process_clip_updt()
 {
     PK_TRACE_DBG("PTH: Clip Updt Entry");
 
@@ -629,7 +657,7 @@ void p9_pgpe_process_clip_updt()
 //
 //p9_pgpe_process_wof_ctrl
 //
-void p9_pgpe_process_wof_ctrl()
+inline void p9_pgpe_process_wof_ctrl()
 {
     PK_TRACE_DBG("PTH: WOF Ctrl Enter");
 
@@ -746,7 +774,7 @@ void p9_pgpe_process_wof_ctrl()
 //
 //p9_pgpe_process_wof_vfrt
 //
-void p9_pgpe_process_wof_vfrt()
+inline void p9_pgpe_process_wof_vfrt()
 {
     PK_TRACE_DBG("PTH: WOF VFRT Enter");
     uint32_t ack_now = 1, bad_rc = 0;
@@ -844,7 +872,7 @@ void p9_pgpe_process_wof_vfrt()
 //
 //p9_pgpe_process_set_pmcr_req
 //
-void p9_pgpe_process_set_pmcr_req()
+inline void p9_pgpe_process_set_pmcr_req()
 {
     PK_TRACE_DBG("PTH: Set PMCR Enter");
 
@@ -944,7 +972,7 @@ void p9_pgpe_process_set_pmcr_req()
     PK_TRACE_DBG("PTH: Set PMCR Exit");
 }
 
-void p9_pgpe_process_registration()
+inline void p9_pgpe_process_registration()
 {
     PK_TRACE_DBG("PTH: Register Enter");
 
@@ -1180,7 +1208,7 @@ void p9_pgpe_process_registration()
 //sent by PGPE to SGPE. This handler is effectively a NOP and is there
 //just to keep the IPC mechanism happy
 //
-void p9_pgpe_process_ack_sgpe_ctrl_stop_updt()
+inline void p9_pgpe_process_ack_sgpe_ctrl_stop_updt()
 {
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_processing = 0;
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_ack = 0;
@@ -1191,8 +1219,44 @@ void p9_pgpe_process_ack_sgpe_ctrl_stop_updt()
 //sent by PGPE to SGPE. This handler is effectively a NOP and is there
 //just to keep the IPC mechanism happy
 //
-void p9_pgpe_process_ack_sgpe_suspend_stop()
+inline void p9_pgpe_process_ack_sgpe_suspend_stop()
 {
+    int q = 0;
+    ocb_qcsr_t qcsr;
+
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_SUSPEND_STOP].pending_processing = 0;
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_SUSPEND_STOP].pending_ack = 0;
+
+
+    //Change PMCR ownership
+    PK_TRACE_INF("SUSP: Setting SCOM Ownership of PMCRs");
+    qcsr.value = in32(OCB_QCSR);
+
+    //Set LMCR for each CME
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
+        {
+            //CME0 within this quad
+            if (qcsr.fields.ex_config & QUAD_EX0_MASK(q))
+            {
+                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 0), BIT64(0));
+            }
+
+            //CME1 within this quad
+            if (qcsr.fields.ex_config & QUAD_EX1_MASK(q))
+            {
+                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 1), BIT64(0));
+            }
+        }
+    }
+
+    //OP Trace and Set OCCS2[PM_COMPLEX_SUSPENDED)
+    p9_pgpe_optrace(ACK_PM_SUSP);
+    uint32_t occScr2 = in32(OCB_OCCS2);
+    occScr2 |= BIT32(PM_COMPLEX_SUSPENDED);
+    G_pgpe_pstate_record.pstatesStatus = PSTATE_PM_SUSPENDED;
+    out32(OCB_OCCS2, occScr2);
+
+    PK_TRACE_INF("SUSP: Suspend Stop Processed");
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_ipc_handlers.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_ipc_handlers.c
index beb9c282..fb377163 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_ipc_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_ipc_handlers.c
@@ -61,61 +61,33 @@ p9_sgpe_ipc_uih_done_hook()
 void
 p9_sgpe_ipc_pgpe_ctrl_stop_updates(ipc_msg_t* cmd, void* arg)
 {
-    PK_TRACE_INF("IPC.PS: Get Control Stop Updates IPC from PGPE");
-
     G_sgpe_stop_record.wof.updates_cmd = cmd;
-
-    // stop in process
-    if (G_sgpe_stop_record.wof.status_stop & STATUS_STOP_PROCESSING)
-    {
-        // Note: response will be sent by stop threads when ongoing stop is completed
-        G_sgpe_stop_record.wof.update_pgpe |= IPC_SGPE_PGPE_UPDATE_CTRL_ONGOING;
-    }
-    // sgpe idle
-    else
-    {
-        p9_sgpe_ack_pgpe_ctrl_stop_updates();
-    }
-}
-
-
-void
-p9_sgpe_ack_pgpe_ctrl_stop_updates()
-{
     ipc_async_cmd_t* async_cmd =
         (ipc_async_cmd_t*)(G_sgpe_stop_record.wof.updates_cmd);
     ipcmsg_p2s_ctrl_stop_updates_t* msg =
         (ipcmsg_p2s_ctrl_stop_updates_t*)async_cmd->cmd_data;
 
-    if (msg->fields.action < 4)
+    if (msg->fields.action == CTRL_STOP_UPDT_ENABLE_CORE)
     {
-        //enable:  core = 0b001, quad = 0b010, both = 0b011
         G_sgpe_stop_record.wof.update_pgpe |= msg->fields.action;
     }
-    else
-    {
-        //disable: core = 0b101, quad = 0b110, both = 0b111
-        G_sgpe_stop_record.wof.update_pgpe &= ~(msg->fields.action);
-    }
-
-    PK_TRACE_INF("IPC.PS: Ack Control Stop Updates IPC to PGPE with Current Action[%x]",
-                 G_sgpe_stop_record.wof.update_pgpe);
 
     msg->fields.return_code  = IPC_SGPE_PGPE_RC_SUCCESS;
-    msg->fields.active_quads = G_sgpe_stop_record.group.quad[VECTOR_ACTIVE] >> SHIFT32(5);
     msg->fields.active_cores = G_sgpe_stop_record.group.core[VECTOR_ACTIVE] >> SHIFT32(23);
-
     ipc_send_rsp(G_sgpe_stop_record.wof.updates_cmd, IPC_RC_SUCCESS);
-    G_sgpe_stop_record.wof.update_pgpe &= ~IPC_SGPE_PGPE_UPDATE_CTRL_ONGOING;
+
+    PK_TRACE_INF("IPC.PS: Ack Control Stop Updates IPC to PGPE with Current Action[%x]",
+                 G_sgpe_stop_record.wof.update_pgpe);
 }
 
 
+
 void
 p9_sgpe_ipc_pgpe_update_active_cores(const uint32_t type)
 {
     uint32_t rc;
 
-    PK_TRACE_INF("IPC.SP: Message PGPE to Update Active Cores with type[%d]", type);
+    PK_TRACE_INF("IPC.SP: Message PGPE to Update Active Cores with type[%d](0/1 EN/EX)", type);
     G_sgpe_ipcmsg_update_cores.fields.update_type = type;
     G_sgpe_ipcmsg_update_cores.fields.return_code = IPC_SGPE_PGPE_RC_NULL;
 
@@ -196,21 +168,25 @@ p9_sgpe_ipc_pgpe_update_active_quads(const uint32_t type, const uint32_t stage)
     uint32_t rc;
 
     G_sgpe_ipcmsg_update_quads.fields.update_type = type;
-    G_sgpe_ipcmsg_update_quads.fields.entry_type = stage;
     G_sgpe_ipcmsg_update_quads.fields.return_code = IPC_SGPE_PGPE_RC_NULL;
 
-    if (type == UPDATE_ACTIVE_CORES_TYPE_EXIT)
+    if (type == UPDATE_ACTIVE_QUADS_TYPE_EXIT)
     {
+        G_sgpe_ipcmsg_update_quads.fields.entry_type = 0;
+        G_sgpe_ipcmsg_update_quads.fields.exit_type = stage;
         G_sgpe_ipcmsg_update_quads.fields.requested_quads =
             G_sgpe_stop_record.group.quad[VECTOR_EXIT] >> SHIFT32(5);
     }
     else
     {
+        G_sgpe_ipcmsg_update_quads.fields.entry_type = stage;
+        G_sgpe_ipcmsg_update_quads.fields.exit_type = 0;
         G_sgpe_ipcmsg_update_quads.fields.requested_quads =
             G_sgpe_stop_record.group.quad[VECTOR_ENTRY] >> SHIFT32(5);
     }
 
-    PK_TRACE_INF("IPC.SP: Message PGPE to Update Active Quads with type[%d][%d], reqQuads=0x%x", type, stage,
+    PK_TRACE_INF("IPC.SP: Message PGPE to Update Active Quads with type[%d](0/1 EN/EX) state[%d](0/1 Done,Notify), reqQuads=0x%x",
+                 type, stage,
                  G_sgpe_ipcmsg_update_quads.fields.requested_quads);
     G_sgpe_ipccmd_to_pgpe_quads.cmd_data = &G_sgpe_ipcmsg_update_quads;
     ipc_init_msg(&G_sgpe_ipccmd_to_pgpe_quads.cmd,
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 23f008c3..9cb875b0 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -221,10 +221,7 @@ enum SGPE_STOP_PSCOM_MASK
 
 enum SGPE_WOF_ACTIVE_UPDATE_STATUS
 {
-    IPC_SGPE_PGPE_UPDATE_QUAD_ENABLED = 0x2,
-    IPC_SGPE_PGPE_UPDATE_CORE_ENABLED = 0x1,
-    // Reserved_4_Do_Not_Use
-    IPC_SGPE_PGPE_UPDATE_CTRL_ONGOING = 0x8,
+    IPC_SGPE_PGPE_UPDATE_CORE_ENABLED = 0x01,
     IPC_SGPE_PGPE_UPDATE_PGPE_HALTED  = 0xF0
 };
 
@@ -386,7 +383,6 @@ struct ring_save
 /// SGPE to PGPE IPC handlers
 void p9_sgpe_ipc_uih_done_hook();
 void p9_sgpe_ipc_pgpe_ctrl_stop_updates(ipc_msg_t*, void*);
-void p9_sgpe_ack_pgpe_ctrl_stop_updates();
 void p9_sgpe_ipc_pgpe_update_active_cores(const uint32_t);
 void p9_sgpe_ipc_pgpe_update_active_cores_poll_ack();
 void p9_sgpe_ipc_pgpe_update_active_quads(const uint32_t, const uint32_t);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index c1f9e107..f8df2305 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -342,9 +342,6 @@ p9_sgpe_stop_exit()
 #if NIMBUS_DD_LEVEL != 10
     uint32_t      fused_core_mode = 0;
 #endif
-#if !SKIP_IPC
-    uint32_t      ipc_exit_quad   = 0;
-#endif
     sgpeHeader_t* pSgpeImgHdr = (sgpeHeader_t*)(OCC_SRAM_SGPE_HEADER_ADDR);
 
     //===============================
@@ -450,12 +447,10 @@ p9_sgpe_stop_exit()
         G_sgpe_stop_record.wof.update_pgpe != IPC_SGPE_PGPE_UPDATE_PGPE_HALTED &&
         G_sgpe_stop_record.group.quad[VECTOR_EXIT])
     {
-        ipc_exit_quad = 1;
-
         p9_sgpe_ipc_pgpe_update_active_quads(UPDATE_ACTIVE_QUADS_TYPE_EXIT,
-                                             UPDATE_ACTIVE_QUADS_ENTRY_TYPE_DONE);
+                                             UPDATE_ACTIVE_QUADS_EXIT_TYPE_NOTIFY);
 
-        /// the poll for ack is located before dpll setup
+        p9_sgpe_ipc_pgpe_update_active_quads_poll_ack(UPDATE_ACTIVE_QUADS_TYPE_EXIT);
     }
 
 #endif
@@ -581,16 +576,6 @@ p9_sgpe_stop_exit()
 #endif
 
 
-#if !SKIP_IPC
-
-        if (ipc_exit_quad)
-        {
-            ipc_exit_quad = 0;
-            p9_sgpe_ipc_pgpe_update_active_quads_poll_ack(UPDATE_ACTIVE_QUADS_TYPE_EXIT);
-        }
-
-#endif
-
         PK_TRACE_DBG("SX.11D: Cache Dpll Setup");
         p9_hcd_cache_dpll_setup(qloop);
 
@@ -1015,6 +1000,21 @@ p9_sgpe_stop_exit()
         p9_sgpe_stop_exit_end(qloop);
     }
 
+
+#if !SKIP_IPC
+
+    if ((in32(OCB_OCCS2) & BIT32(PGPE_ACTIVE)) &&
+        G_sgpe_stop_record.wof.update_pgpe != IPC_SGPE_PGPE_UPDATE_PGPE_HALTED &&
+        G_sgpe_stop_record.group.quad[VECTOR_EXIT])
+    {
+        p9_sgpe_ipc_pgpe_update_active_quads(UPDATE_ACTIVE_QUADS_TYPE_EXIT,
+                                             UPDATE_ACTIVE_QUADS_EXIT_TYPE_DONE);
+
+        p9_sgpe_ipc_pgpe_update_active_quads_poll_ack(UPDATE_ACTIVE_QUADS_TYPE_EXIT);
+    }
+
+#endif
+
     //===========================
     MARK_TRAP(ENDSCOPE_STOP_EXIT)
     //===========================
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
index 235b938c..c09440d6 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
@@ -113,6 +113,9 @@ p9_sgpe_pgpe_halt_handler(void* arg, PkIrqId irq)
 {
     PkMachineContext   ctx;
 
+    g_oimr_override |=   BIT64(7);
+    out32(OCB_OIMR0_OR,  BIT32(7));
+
     PK_TRACE_INF("WARNING: PGPE Halted Due to Error");
     PK_OPTIONAL_DEBUG_HALT(SGPE_PGPE_ERROR_DETECTED);
     out32(OCB_OISR0_CLR, BIT32(7));
@@ -129,6 +132,9 @@ p9_sgpe_checkstop_handler(void* arg, PkIrqId irq)
 {
     PkMachineContext   ctx;
 
+    g_oimr_override |=   BIT64(16);
+    out32(OCB_OIMR0_OR,  BIT32(16));
+
     PK_TRACE_INF("WARNING: System Checkstop Detected");
     PK_OPTIONAL_DEBUG_HALT(SGPE_SYSTEM_CHECKSTOP_DETECTED);
     out32(OCB_OISR0_CLR, BIT32(16));
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_threads.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_threads.c
index e5e1c095..e57b0c25 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_threads.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_threads.c
@@ -57,11 +57,6 @@ p9_sgpe_stop_exit_thread(void* arg)
 
 #endif
 
-            if (G_sgpe_stop_record.wof.update_pgpe & IPC_SGPE_PGPE_UPDATE_CTRL_ONGOING)
-            {
-                p9_sgpe_ack_pgpe_ctrl_stop_updates();
-            }
-
             PK_TRACE_INF("Setup: Exit Done,no Entry Request.Enable Type0/2/3/6 Interrupt");
             g_oimr_override &= ~(BIT64(45) | BITS64(47, 2) | BIT64(51));
             pk_irq_vec_restore(&ctx);
@@ -99,11 +94,6 @@ p9_sgpe_stop_enter_thread(void* arg)
 
 #endif
 
-        if (G_sgpe_stop_record.wof.update_pgpe & IPC_SGPE_PGPE_UPDATE_CTRL_ONGOING)
-        {
-            p9_sgpe_ack_pgpe_ctrl_stop_updates();
-        }
-
         PK_TRACE_INF("Setup: Entry done. Enable Type0/2/3/6 Interrupt");
         g_oimr_override &= ~(BIT64(45) | BITS64(47, 2) | BIT64(51));
         pk_irq_vec_restore(&ctx);