CME/SGPE: Complete Working STOP8 CME/SGPE Images Snapshot

Change-Id: Ib2d67d71fbee16b250bd6f2143b0c8b4d82c4cf2
Original-Change-Id: I5e8cc9cbdccb10edcc4e2474f1c9a15e41f2c6ed
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/23292
Tested-by: Jenkins Server
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

3 files changed, 642 insertions, 477 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index 24c3ed87..bb0c164f 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -46,13 +46,16 @@
 #include "p9_cme_stop.h"
 #include "p9_cme_stop_enter_marks.h"
 
-CmeStopRecord G_cme_stop_record = {0, 0, 0};
+extern CmeStopRecord G_cme_stop_record;
 
 int
 p9_cme_stop_entry()
 {
 
     int             rc                  = 0;
+    uint8_t         core_aborted        = 0;
+    uint8_t         core_catchup        = 0;
+    uint8_t         catchup_ongoing     = 0;
     uint8_t         entry_ongoing       = 1;
     uint8_t         target_level;
     uint8_t         deeper_level        = 0;
@@ -73,6 +76,7 @@ p9_cme_stop_entry()
     // STOP PM_STATE. If both have fired by the time we get to this point,
     // CME will do Daul-cast to both cores at the same time in entry flow.
     core = (in32(CME_LCL_EISR) & BITS32(20, 2)) >> SHIFT32(21);
+
     // override with partial good core mask
     core = core & G_cme_stop_record.cme_flags & CME_MASK_BC;
 
@@ -88,62 +92,49 @@ p9_cme_stop_entry()
     MARK_TAG(BEGINSCOPE_STOP_ENTRY, core)
     //===================================
 
-    //--------------------------------------------------------------------------
-    // STOP LEVEL 1 (should be done by hardware)
-    //--------------------------------------------------------------------------
-
-    // Read SISR for pm_state_cX
-    pm_states = in32_sh(CME_LCL_SISR);
-    G_cme_stop_record.pm_state_c0 = (pm_states & BITS32(4, 4)) >> SHIFT32(7);
-    G_cme_stop_record.pm_state_c1 = (pm_states & BITS32(8, 4)) >> SHIFT32(11);
-#if LAB_MODE == 1
-
-    if (G_cme_stop_record.pm_state_c0 == STOP_LEVEL_11)
-    {
-        G_cme_stop_record.pm_state_c0 = STOP_LEVEL_9;
-    }
-
-    if (G_cme_stop_record.pm_state_c1 == STOP_LEVEL_11)
+    do   // while(0) loop for stop flow control
     {
-        G_cme_stop_record.pm_state_c1 = STOP_LEVEL_9;
-    }
 
-#endif
+        //--------------------------------------------------------------------------
+        // STOP LEVEL 1 (should be done by hardware)
+        //--------------------------------------------------------------------------
 
-    // pm_active AND waken_up   : pm_state = sisr (new state)
-    // pm_active AND !waken_up  : pm_state = sisr (only with auto promote)
-    // !pm_active AND waken_up  : pm_state = 0    (state out of date)
-    // !pm_active AND !waken_up : pm_state = sisr (current state)
-    if (~core & G_cme_stop_record.cme_wakenup & CME_MASK_C0)
-    {
-        G_cme_stop_record.pm_state_c0 = 0;
-    }
+        // Read SISR for pm_state_cX
+        pm_states = in32_sh(CME_LCL_SISR);
+        G_cme_stop_record.req_stop_c0 = (pm_states & BITS32(4, 4)) >> SHIFT32(7);
+        G_cme_stop_record.req_stop_c1 = (pm_states & BITS32(8, 4)) >> SHIFT32(11);
 
-    if (~core & G_cme_stop_record.cme_wakenup & CME_MASK_C1)
-    {
-        G_cme_stop_record.pm_state_c1 = 0;
-    }
+        // pm_active AND waken_up   : pm_state = sisr (new state)
+        // pm_active AND !waken_up  : pm_state = sisr (only with auto promote)
+        // !pm_active AND waken_up  : pm_state = 0    (out dated state)
+        // !pm_active AND !waken_up : pm_state = sisr (current state)
+        if (~core & G_cme_stop_record.active_core & CME_MASK_C0)
+        {
+            G_cme_stop_record.req_stop_c0 = 0;
+        }
 
-    G_cme_stop_record.cme_wakenup &= ~core;
+        if (~core & G_cme_stop_record.active_core & CME_MASK_C1)
+        {
+            G_cme_stop_record.req_stop_c1 = 0;
+        }
 
-    PK_TRACE("SE1: Stop Levels[%d %d]",
-             G_cme_stop_record.pm_state_c0, G_cme_stop_record.pm_state_c1);
+        G_cme_stop_record.active_core &= ~core;
 
-    // Return error if target STOP level == 1(Nap)
-    if((core == CME_MASK_C0 && G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_1) ||
-       (core == CME_MASK_C1 && G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_1) ||
-       (core == CME_MASK_BC && (G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_1 ||
-                                G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_1)))
-    {
-        return CME_STOP_ENTRY_STOP1_SENT_IRQ;
-    }
+        PK_TRACE("SE1: Request Stop Levels[%d %d]",
+                 G_cme_stop_record.req_stop_c0, G_cme_stop_record.req_stop_c1);
 
-    //--------------------------------------------------------------------------
-    // STOP LEVEL 2
-    //--------------------------------------------------------------------------
+        // Return error if target STOP level == 1(Nap)
+        if((core == CME_MASK_C0 && G_cme_stop_record.req_stop_c0 <= STOP_LEVEL_1) ||
+           (core == CME_MASK_C1 && G_cme_stop_record.req_stop_c1 <= STOP_LEVEL_1) ||
+           (core == CME_MASK_BC && (G_cme_stop_record.req_stop_c0 <= STOP_LEVEL_1 ||
+                                    G_cme_stop_record.req_stop_c1 <= STOP_LEVEL_1)))
+        {
+            return CME_STOP_ENTRY_STOP1_SENT_IRQ;
+        }
 
-    do   // while(0) loop for stop flow control
-    {
+        //--------------------------------------------------------------------------
+        // STOP LEVEL 2
+        //--------------------------------------------------------------------------
 
         PK_TRACE("SE2.a");
         // Disable fired Stop and corresponding Wakeup interrupts
@@ -165,22 +156,22 @@ p9_cme_stop_entry()
         CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(0));
 
         // set target_level from pm_state for both cores or just one core
-        target_level = (core == CME_MASK_C0) ? G_cme_stop_record.pm_state_c0 :
-                       G_cme_stop_record.pm_state_c1;
+        target_level = (core == CME_MASK_C0) ? G_cme_stop_record.req_stop_c0 :
+                       G_cme_stop_record.req_stop_c1;
 
         // If both cores are going into STOP but targeting different levels,
         if ((core == CME_MASK_BC) &&
-            (G_cme_stop_record.pm_state_c0 != G_cme_stop_record.pm_state_c1))
+            (G_cme_stop_record.req_stop_c0 != G_cme_stop_record.req_stop_c1))
         {
             // set target_level to the lighter level targeted by one core
             // set deeper_level to the deeper level targeted by deeper core
-            deeper_level = G_cme_stop_record.pm_state_c0;
+            deeper_level = G_cme_stop_record.req_stop_c0;
             deeper_core  = CME_MASK_C0;
 
-            if (G_cme_stop_record.pm_state_c0 < G_cme_stop_record.pm_state_c1)
+            if (G_cme_stop_record.req_stop_c0 < G_cme_stop_record.req_stop_c1)
             {
-                target_level = G_cme_stop_record.pm_state_c0;
-                deeper_level = G_cme_stop_record.pm_state_c1;
+                target_level = G_cme_stop_record.req_stop_c0;
+                deeper_level = G_cme_stop_record.req_stop_c1;
                 deeper_core  = CME_MASK_C1;
             }
         }
@@ -239,7 +230,7 @@ p9_cme_stop_entry()
         PK_TRACE("SE2.i");
         // Waits quiesce done for at least 512 core cycles
         // MF: verify generate FCB otherwise math is wrong.
-        CME_WAIT_CORE_CYCLES(loop, 512)
+        PPE_WAIT_CORE_CYCLES(loop, 512)
 
         PK_TRACE("SE2.j");
         // Raise Core Chiplet Fence
@@ -289,7 +280,7 @@ p9_cme_stop_entry()
 
         PK_TRACE("SE2.p");
         // Switch glsmux to refclk to save clock grid power
-        CME_PUTSCOM(PPM_CGCR_CLR, core, BIT64(3));
+        CME_PUTSCOM(C_PPM_CGCR, core, BIT64(3));
 
         // Assert PCB Fence
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
@@ -327,6 +318,16 @@ p9_cme_stop_entry()
             entry_ongoing = 1;
         }
 
+        if (core & CME_MASK_C0)
+        {
+            G_cme_stop_record.act_stop_c0 = STOP_LEVEL_2;
+        }
+
+        if (core & CME_MASK_C1)
+        {
+            G_cme_stop_record.act_stop_c1 = STOP_LEVEL_2;
+        }
+
         PK_TRACE("SE2: Clock Sync Dropped");
 
         //===========================
@@ -336,13 +337,80 @@ p9_cme_stop_entry()
         //===========================
         MARK_TRAP(SE_IS0_BEGIN)
         //===========================
-        // todo: open wakeup interrupt window if we are not done
-        // todo: open catchup interrupt window if we are not done
+#if !SKIP_ABORT
+        out32(CME_LCL_EIMR_CLR, /*TODO(core << SHIFT32(13)) |*/
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+        sync();
+        out32(CME_LCL_EIMR_OR,  (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+#endif
         //===================
         MARK_TRAP(SE_IS0_END)
         //===================
 
-        // If we are done at STOP level 2
+        core_aborted = 0;
+
+        if ((core & CME_MASK_C0) &&
+            G_cme_stop_record.act_stop_c0 == STOP_LEVEL_0 &&
+            G_cme_stop_record.req_stop_c0 != STOP_LEVEL_0)
+        {
+            core_aborted |= CME_MASK_C0;
+            core -= CME_MASK_C0;
+            deeper_core = 0;
+
+            if (deeper_core == CME_MASK_C1)
+            {
+                target_level = deeper_level;
+            }
+
+        }
+
+        if ((core & CME_MASK_C1) &&
+            G_cme_stop_record.act_stop_c1 == STOP_LEVEL_0 &&
+            G_cme_stop_record.req_stop_c1 != STOP_LEVEL_0)
+        {
+            core_aborted |= CME_MASK_C1;
+            core -= CME_MASK_C1;
+            deeper_core = 0;
+
+            if (deeper_core == CME_MASK_C0)
+            {
+                target_level = deeper_level;
+            }
+        }
+
+        if (!core)
+        {
+            core |= core_aborted;
+            entry_ongoing = 0;
+            break;
+        }
+
+        if (catchup_ongoing)
+        {
+            core = CME_MASK_BC;
+            break;
+        }
+
+#if !SKIP_CATCHUP
+        core_catchup = (in32(CME_LCL_EISR) & BITS32(20, 2)) >> SHIFT32(21);
+#endif
+
+        if (core_catchup > core)
+        {
+            core = core_catchup - core;
+            catchup_ongoing = 1;
+        }
+
+    }
+    while(catchup_ongoing);
+
+    do
+    {
+
+        // If we are done at STOP level 2 or aborted
         if (!entry_ongoing)
         {
             break;
@@ -403,6 +471,16 @@ p9_cme_stop_entry()
                 entry_ongoing = 0;
             }
 
+            if(core & CME_MASK_C0)
+            {
+                G_cme_stop_record.act_stop_c0 = STOP_LEVEL_3;
+            }
+
+            if(core & CME_MASK_C1)
+            {
+                G_cme_stop_record.act_stop_c1 = STOP_LEVEL_3;
+            }
+
             //===========================
             MARK_TAG(SE_STOP3_DONE, core)
             //===========================
@@ -418,185 +496,224 @@ p9_cme_stop_entry()
         // STOP LEVEL 4
         //--------------------------------------------------------------------------
 
-        // Skip Power off completely only if single or both core target STOP L9
-        // that is target_level == 9 and deeper_core == 0
-        if (target_level != 9 || (deeper_core && deeper_level != 9))
-        {
-
-            //===============================
-            MARK_TAG(SE_POWER_OFF_CORE, core)
-            //===============================
-
-            // We may skip one of the cores in deeper_core case
-            if (deeper_core)
-            {
-                if (deeper_level == 9)
-                {
-                    core = deeper_core ^ CME_MASK_BC;
-                }
-
-                if (target_level == 9)
-                {
-                    core = deeper_core;
-                }
-            }
+        //===============================
+        MARK_TAG(SE_POWER_OFF_CORE, core)
+        //===============================
 
 #if !STOP_PRIME
-            // Assert Cores Electrical Fences
-            PK_TRACE("SE4.a");
-            CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(26));
+        // Assert Cores Electrical Fences
+        PK_TRACE("SE4.a");
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(26));
 
-            // Make sure we are not forcing PFET for VDD off
-            // vdd_pfet_force_state == 00 (Nop)
-            PK_TRACE("SE4.b");
-            CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+        // Make sure we are not forcing PFET for VDD off
+        // vdd_pfet_force_state == 00 (Nop)
+        PK_TRACE("SE4.b");
+        CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
 
-            if (scom_data & BITS64(0, 2))
-            {
-                return CME_STOP_ENTRY_VDD_PFET_NOT_IDLE;
-            }
+        if (scom_data & BITS64(0, 2))
+        {
+            return CME_STOP_ENTRY_VDD_PFET_NOT_IDLE;
+        }
 
-            // Prepare PFET Controls
-            // vdd_pfet_val/sel_override     = 0 (disbaled)
-            // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
-            PK_TRACE("SE4.c");
-            CME_PUTSCOM(PPM_PFCS_CLR, core, BIT64(4) | BIT64(5) | BIT64(8));
+        // Prepare PFET Controls
+        // vdd_pfet_val/sel_override     = 0 (disbaled)
+        // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
+        PK_TRACE("SE4.c");
+        CME_PUTSCOM(PPM_PFCS_CLR, core, BIT64(4) | BIT64(5) | BIT64(8));
 
-            // Power Off Core VDD
-            // vdd_pfet_force_state = 01 (Force Voff)
-            PK_TRACE("SE4.d");
-            CME_PUTSCOM(PPM_PFCS_OR, core, BIT64(1));
+        // Power Off Core VDD
+        // vdd_pfet_force_state = 01 (Force Voff)
+        PK_TRACE("SE4.d");
+        CME_PUTSCOM(PPM_PFCS_OR, core, BIT64(1));
 
-            // Poll for power gate sequencer state: 0x8 (FSM Idle)
-            PK_TRACE("SE4.e");
+        // Poll for power gate sequencer state: 0x8 (FSM Idle)
+        PK_TRACE("SE4.e");
 
-            do
-            {
-                CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
-            }
-            while(!(scom_data & BIT64(42)));
+        do
+        {
+            CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+        }
+        while(!(scom_data & BIT64(42)));
 
 #if !EPM_P9_TUNING
-            // Optional: Poll for vdd_pg_sel being: 0x8
-            PK_TRACE("SE4.f");
+        // Optional: Poll for vdd_pg_sel being: 0x8
+        PK_TRACE("SE4.f");
 
-            do
-            {
-                CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
-            }
-            while(!(scom_data & BIT64(46)));
+        do
+        {
+            CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+        }
+        while(!(scom_data & BIT64(46)));
 
 #endif
 
-            // Turn Off Force Voff
-            // vdd_pfet_force_state = 00 (Nop)
-            PK_TRACE("SE4.g");
-            CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
+        // Turn Off Force Voff
+        // vdd_pfet_force_state = 00 (Nop)
+        PK_TRACE("SE4.g");
+        CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
 #endif
 
-            PK_TRACE("SE4.h");
-            // Update Stop History: In Core Stop Level 4
-            // Check if STOP level 4 reaches the target for both or one core
-            entry_ongoing =
-                target_level == STOP_LEVEL_4 ? STOP_TRANS_COMPLETE :
-                STOP_TRANS_ENTRY;
-            CME_STOP_UPDATE_HISTORY(core,
+        PK_TRACE("SE4.h");
+        // Update Stop History: In Core Stop Level 4
+        // Check if STOP level 4 reaches the target for both or one core
+        entry_ongoing =
+            target_level == STOP_LEVEL_4 ? STOP_TRANS_COMPLETE :
+            STOP_TRANS_ENTRY;
+        CME_STOP_UPDATE_HISTORY(core,
+                                STOP_CORE_IS_GATED,
+                                entry_ongoing,
+                                target_level,
+                                STOP_LEVEL_4,
+                                STOP_REQ_DISABLE,
+                                STOP_ACT_ENABLE);
+
+        // If both cores targeting different levels
+        // deeper core should have at least deeper stop level than 2
+        // only need to modify deeper core history if another one was done
+        if (deeper_core && !entry_ongoing)
+        {
+            CME_STOP_UPDATE_HISTORY(deeper_core,
                                     STOP_CORE_IS_GATED,
-                                    entry_ongoing,
-                                    target_level,
+                                    STOP_TRANS_ENTRY,
+                                    deeper_level,
                                     STOP_LEVEL_4,
                                     STOP_REQ_DISABLE,
-                                    STOP_ACT_ENABLE);
+                                    STOP_ACT_DISABLE);
+            // from now on, proceed with only deeper core
+            core          = deeper_core;
+            target_level  = deeper_level;
+            deeper_level  = 0;
+            deeper_core   = 0;
+            entry_ongoing = 1;
+        }
 
-            // restore back to both cores in deeper_core case
-            if (deeper_core)
-            {
-                core = CME_MASK_BC;
-            }
+        if (core & CME_MASK_C0)
+        {
+            G_cme_stop_record.act_stop_c0 = STOP_LEVEL_4;
+        }
 
-            // If both cores targeting different levels
-            // deeper core should have at least deeper stop level than 2
-            // only need to modify deeper core history if another one was done
-            if (deeper_core && !entry_ongoing)
+        if (core & CME_MASK_C1)
+        {
+            G_cme_stop_record.act_stop_c1 = STOP_LEVEL_4;
+        }
+
+        PK_TRACE("SE4: Core Powered Off");
+
+        //===========================
+        MARK_TAG(SE_STOP4_DONE, core)
+        //===========================
+
+        //===========================
+        MARK_TRAP(SE_IS1_BEGIN)
+        //===========================
+#if !SKIP_ABORT
+        out32(CME_LCL_EIMR_CLR, /*TODO(core << SHIFT32(13)) |*/
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+        sync();
+        out32(CME_LCL_EIMR_OR,  (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+#endif
+        //===================
+        MARK_TRAP(SE_IS1_END)
+        //===================
+
+        if ((core & CME_MASK_C0) &&
+            G_cme_stop_record.act_stop_c0 == STOP_LEVEL_0 &&
+            G_cme_stop_record.req_stop_c0 != STOP_LEVEL_0)
+        {
+            core_aborted |= CME_MASK_C0;
+            core -= CME_MASK_C0;
+            deeper_core = 0;
+
+            if (deeper_core == CME_MASK_C1)
             {
-                CME_STOP_UPDATE_HISTORY(deeper_core,
-                                        STOP_CORE_IS_GATED,
-                                        STOP_TRANS_ENTRY,
-                                        deeper_level,
-                                        STOP_LEVEL_4,
-                                        STOP_REQ_DISABLE,
-                                        STOP_ACT_DISABLE);
-                // from now on, proceed with only deeper core
-                core          = deeper_core;
-                target_level  = deeper_level;
-                deeper_level  = 0;
-                deeper_core   = 0;
-                entry_ongoing = 1;
+                target_level = deeper_level;
             }
 
-            PK_TRACE("SE4: Core Powered Off");
-
-            //===========================
-            MARK_TAG(SE_STOP4_DONE, core)
-            //===========================
+        }
 
-            //===========================
-            MARK_TRAP(SE_IS1_BEGIN)
-            //===========================
-            // todo: open wakeup interrupt window if we are not done
-            //===================
-            MARK_TRAP(SE_IS1_END)
-            //===================
+        if ((core & CME_MASK_C1) &&
+            G_cme_stop_record.act_stop_c1 == STOP_LEVEL_0 &&
+            G_cme_stop_record.req_stop_c1 != STOP_LEVEL_0)
+        {
+            core_aborted |= CME_MASK_C1;
+            core -= CME_MASK_C1;
+            deeper_core = 0;
 
-            // If we are done at STOP level 4
-            if (!entry_ongoing)
+            if (deeper_core == CME_MASK_C0)
             {
-                break;
+                target_level = deeper_level;
             }
+        }
 
-            PK_TRACE("SE4+:core[%d],deeper_core[%d],\
-                      target_level[%d],deeper_level[%d]",
-                     core, deeper_core, target_level, deeper_level);
+        if (!core)
+        {
+            core |= core_aborted;
+            entry_ongoing = 0;
         }
 
+        // If we are done at STOP level 4 or aborted
+        if (!entry_ongoing)
+        {
+            break;
+        }
+
+        PK_TRACE("SE4+:core[%d],deeper_core[%d],\
+                  target_level[%d],deeper_level[%d]",
+                 core, deeper_core, target_level, deeper_level);
+
         //--------------------------------------------------------------------------
         // STOP LEVEL 5 (preparsion of STOP LEVEL 8 and above)
         //--------------------------------------------------------------------------
 
-        if ((G_cme_stop_record.pm_state_c0 >= STOP_LEVEL_8) &&
-            (G_cme_stop_record.pm_state_c1 >= STOP_LEVEL_8))
+        // block all wake up before purge L2,
+        // this is last chance either core can exit
+        out32(CME_LCL_EIMR_OR, BITS32(12, 6));
+
+        if ((G_cme_stop_record.req_stop_c0 >= STOP_LEVEL_8) &&
+            (G_cme_stop_record.req_stop_c1 >= STOP_LEVEL_8))
         {
 
             //=========================
             MARK_TAG(SE_PURGE_L2, core)
             //=========================
 
-            // Assert L2+NCU Purges(chtm purge will be done in SGPE), and NCU tlbie quiesce
+            // Assert L2+NCU Purges(chtm purge will be done in SGPE),
+            // and NCU tlbie quiesce
             PK_TRACE("SE5.1a");
             out32(CME_LCL_SICR_OR, BIT32(18) | BIT32(21) | BIT32(22));
 
+            // todo: poll for tlbie quiesce done?
+
+            // Poll for Purged Done
+            PK_TRACE("SE5.1b");
             //===========================
             MARK_TRAP(SE_IS2_BEGIN)
-            //===========================
 
-            // todo: open wakeup window to abort purge
+            //===========================
+            do
+            {
+#if !SKIP_ABORT
 
-            // Poll for Purged Done
-            PK_TRACE("SE5.1b");
+                if(in32(CME_LCL_EINR) & BITS32(12, 6))
+                {
+                    // abort L2+NCU purges
+                    out32(CME_LCL_SICR_OR, BIT32(19) | BIT32(23));
+                }
 
+#endif
+            }
             while((in32(CME_LCL_EISR) & BITS32(22, 2)) != BITS32(22, 2));
 
-            // todo: close wakeup window
-
             //===================
             MARK_TRAP(SE_IS2_END)
             //===================
 
-            // Deassert L2+NCU Purges
+            // Deassert L2+NCU Purges, their possible aborts, NCU tlbie quiesce
             PK_TRACE("SE5.1c");
-            out32(CME_LCL_SICR_CLR, BIT32(18) | BIT32(22));
-
-            // Raise L2-L3 quiesce???
+            out32(CME_LCL_SICR_CLR, BITS32(18, 6));
 
             PK_TRACE("SE5.1: L2/NCU/CHTM Purged");
 
@@ -605,15 +722,6 @@ p9_cme_stop_entry()
             //=========================
         }
 
-
-        // Recheck status in case L2 purge is aborted by wakeup
-        if ((G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_4) &&
-            (G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_4))
-        {
-            entry_ongoing = 0;
-            break;
-        }
-
         //=============================
         MARK_TAG(SE_SGPE_HANDOFF, core)
         //=============================
@@ -628,19 +736,29 @@ p9_cme_stop_entry()
                                 STOP_REQ_DISABLE,
                                 STOP_ACT_ENABLE);
 
+        if (core & CME_MASK_C0)
+        {
+            G_cme_stop_record.act_stop_c0 = STOP_LEVEL_5;
+        }
+
+        if (core & CME_MASK_C1)
+        {
+            G_cme_stop_record.act_stop_c1 = STOP_LEVEL_5;
+        }
+
         // Send PCB Interrupt per core
         PK_TRACE("SE5.2b");
         pig.fields.req_intr_type = 2; //0b010: STOP State Change
 
         if (core & CME_MASK_C0)
         {
-            pig.fields.req_intr_payload = G_cme_stop_record.pm_state_c0;
+            pig.fields.req_intr_payload = G_cme_stop_record.req_stop_c0;
             CME_PUTSCOM(PPM_PIG, CME_MASK_C0, pig.value);
         }
 
         if (core & CME_MASK_C1)
         {
-            pig.fields.req_intr_payload = G_cme_stop_record.pm_state_c1;
+            pig.fields.req_intr_payload = G_cme_stop_record.req_stop_c1;
             CME_PUTSCOM(PPM_PIG, CME_MASK_C1, pig.value);
         }
 
@@ -660,16 +778,20 @@ p9_cme_stop_entry()
     // Release PPM Write Protection
     CME_PUTSCOM(CPPM_CPMMR_CLR, core, BIT64(0));
 
-    // Enable fired Stop and corresponding Wakeup Interrupts
-    // if no handoff to SGPE
-    //this exit is umasked, this entry remains masked
-    //other entry is umasked only if it's waken up
+    // Enable Wakeup Interrupts of this entry core if no handoff to SGPE
+    // Otherwise enable Doorbell interrupts of this entry core
+    out32(CME_LCL_EIMR_CLR,
+          ((~core & G_cme_stop_record.active_core & CME_MASK_BC) << SHIFT32(21)));
+
     if (!entry_ongoing)
         out32(CME_LCL_EIMR_CLR,
-              ((~core & CME_MASK_BC & G_cme_stop_record.cme_wakenup) << SHIFT32(21)) |
-              (core << SHIFT32(13)) |
-              (core << SHIFT32(15)) |
-              (core << SHIFT32(17)));
+              /*TODO((core & ~core_aborted & CME_MASK_BC) << SHIFT32(13)) |*/
+              ((core & ~core_aborted & CME_MASK_BC) << SHIFT32(15)) |
+              ((core & ~core_aborted & CME_MASK_BC) << SHIFT32(17)));
+    else
+    {
+        out32_sh(CME_LCL_EIMR_CLR, (core << SHIFT32((41 - 32))));
+    }
 
     //============================
     MARK_TRAP(ENDSCOPE_STOP_ENTRY)
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 0febd155..359f8fe9 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -49,39 +49,42 @@
 
 #include "p9_stop_common.h"
 
+#define EQ_OPCG_ALIGN            0x10030001
 #define EQ_SCAN_REGION_TYPE      0x10030005
 #define EQ_CLK_REGION            0x10030006
 #define EQ_CLOCK_STAT_SL         0x10030008
+
+#define EQ_BIST                  0x100F000B
 #define EQ_NET_CTRL0_WAND        0x100F0041
 #define EQ_NET_CTRL0_WOR         0x100F0042
-#define EQ_CPLT_CTRL0_CLEAR      0x10000020
+
 #define EQ_CPLT_CTRL0_OR         0x10000010
-#define EQ_CPLT_CTRL1_CLEAR      0x10000021
+#define EQ_CPLT_CTRL0_CLEAR      0x10000020
 #define EQ_CPLT_CTRL1_OR         0x10000011
-#define EQ_BIST                  0x100F000B
-#define PM_PURGE_REG             0x10011C13
-#define DRAM_REF_REG             0x10011C0F
-#define EQ_QPPM_EDRAM_CTRL_CLEAR 0x100F01BE
-#define EQ_QPPM_EDRAM_CTRL_OR    0x100F01BF
+#define EQ_CPLT_CTRL1_CLEAR      0x10000021
+
 #define EQ_QPPM_DPLL_CTRL_CLEAR  0x100F0153
 #define EQ_QPPM_DPLL_CTRL_OR     0x100F0154
 #define EQ_QPPM_DPLL_STAT        0x100F0155
-#define EQ_PPM_CGCR_CLEAR        0x100F0166
-#define EQ_PPM_CGCR_OR           0x100F0167
-#define EQ_QPPM_QCCR             0x100F01BD
-#define EQ_QPPM_QCCR_WCLEAR      0x100F01BE
-#define EQ_QPPM_QCCR_WOR         0x100F01BF
 #define EQ_QPPM_QACCR_SCOM1      0x100F0161
 #define EQ_QPPM_QACCR_SCOM2      0x100F0162
 #define EQ_QPPM_QACSR            0x100F0163
+#define EQ_PPM_CGCR              0x100F0164
 #define EQ_QPPM_EXCGCR           0x100F0165
 #define EQ_QPPM_EXCGCR_CLR       0x100F0166
 #define EQ_QPPM_EXCGCR_OR        0x100F0167
-#define EQ_PM_LCO_DIS_REG        0x0
-#define EQ_PM_L2_RCMD_DIS_REG    0x0
+#define EQ_QPPM_QCCR             0x100F01BD
+#define EQ_QPPM_QCCR_WCLEAR      0x100F01BE
+#define EQ_QPPM_QCCR_WOR         0x100F01BF
+
+#define EX_DRAM_REF_REG          0x1001180F
+#define EX_PM_PURGE_REG          0x10011813
+#define EX_PM_LCO_DIS_REG        0x10011816
+#define EX_PM_L2_RCMD_DIS_REG    0x10011818
 
 #define SGPE_STOP_L2_CLOCK_REGION(ex) (ex << SHIFT64(9))
 #define SGPE_STOP_L3_CLOCK_REGION(ex) (ex << SHIFT64(7))
+
 /// Macro to update STOP History
 #define SGPE_STOP_UPDATE_HISTORY(id,base,gated,trans,req_l,act_l,req_e,act_e) \
     hist.fields.stop_gated       = gated;                                     \
@@ -90,7 +93,7 @@
     hist.fields.act_stop_level   = act_l;                                     \
     hist.fields.req_write_enable = req_e;                                     \
     hist.fields.act_write_enable = act_e;                                     \
-    GPE_PUTSCOM(PPM_SSHSRC, base, id, hist.value);
+    GPE_PUTSCOM_VAR(PPM_SSHSRC, base, id, 0, hist.value);
 
 
 enum SGPE_STOP_RETURN_CODES
@@ -116,7 +119,7 @@ enum SGPE_STOP_IRQ_PAYLOAD_MASKS
 enum SGPE_STOP_EVENT_LEVELS
 {
     SGPE_EX_BASE_LV                   = 8,
-    SGPE_EQ_BASE_LV                   = 9
+    SGPE_EQ_BASE_LV                   = 11
 };
 
 enum SGPE_STOP_EVENT_FLAGS
@@ -147,11 +150,6 @@ typedef union sgpe_state
     uint32_t status;
     struct
     {
-        uint16_t target : 16;
-        uint16_t actual : 16;
-    } differ;
-    struct
-    {
         uint8_t  spare0 : 4;
         uint8_t  q_req  : 4;
         uint8_t  x0req  : 4;
@@ -168,18 +166,12 @@ typedef union sgpe_group
     uint64_t vector[2];
     struct
     {
-        uint32_t exit;
-        uint32_t entry;
-        uint32_t xq_in;
-        uint32_t spare;
-    } action;
-    struct
-    {
         uint32_t c_out;
+        uint16_t x_out;
+        uint16_t q_out;
         uint32_t c_in;
         uint16_t x_in;
         uint16_t q_in;
-        uint32_t spare;
     } member;
 } sgpe_group_t;
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index 4e53baa2..875c33be 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -36,160 +36,168 @@ p9_sgpe_stop_entry()
     uint8_t      qloop;
     uint8_t      cloop;
     uint8_t      climit;
-    uint8_t      qentry;
+    uint16_t     qentry;
     uint16_t     xentry;
-    uint64_t     scom_data;
     uint8_t      entry_ongoing[2] = {0, 0};
+    uint8_t      l3_purge_aborted = 0;
+    uint32_t     loop;
+    uint64_t     scom_data;
     ppm_sshsrc_t hist;
 
     //===================================================================
-    MARK_TAG(BEGINSCOPE_STOP_ENTRY, G_sgpe_stop_record.group.member.c_in)
+    MARK_TAG(BEGINSCOPE_STOP_ENTRY, (G_sgpe_stop_record.group.member.q_in >> 10))
     //===================================================================
 
-    // ------------------------------------------------------------------------
-    // EX STOP ENTRY
-    // ------------------------------------------------------------------------
+    if (G_sgpe_stop_record.group.member.q_in)
+        MARK_TRAP(SE_STOP_SUSPEND_PSTATE)
 
-    for(xentry = G_sgpe_stop_record.group.member.x_in, qloop = 0;
-        xentry > 0;
-        xentry = xentry << 2, qloop++)
-    {
-        // if this ex is not up to entry, skip
-        if (!(ex = ((xentry & BITS16(0, 2)) >> SHIFT16(1))))
-        {
-            continue;
-        }
-
-        PK_TRACE("q[%d]exmask[%d] starts entry", qloop, ex);
-
-        //-------------------------------------------------------------------------
-        // STOP LEVEL 8
-        //-------------------------------------------------------------------------
-
-        // Update QSSR: stop_entry_ongoing
-        out32(OCB_QSSR_OR, BIT32(qloop + 20));
+        //TODO: message pgpe to suspend Pstate
 
-        // Update History for ongoing stop 8 entry
-        cloop  = (ex & FST_EX_IN_QUAD) ?              0 : CORES_PER_EX;
-        climit = (ex & SND_EX_IN_QUAD) ? CORES_PER_QUAD : CORES_PER_EX;
-
-        for(; cloop < climit; cloop++)
-        {
-            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
-                                     CORE_ADDR_BASE,
-                                     STOP_CORE_IS_GATED,
-                                     STOP_TRANS_ENTRY,
-                                     STOP_LEVEL_8,
-                                     STOP_LEVEL_8,
-                                     STOP_REQ_DISABLE,
-                                     STOP_ACT_DISABLE);
-        }
-
-        //========================
-        MARK_TRAP(SE_STOP_L2_CLKS)
-        //========================
-
-        PK_TRACE("SE8.a");
-        // Disable L2 Snoop(quiesce L2-L3 interface, what about NCU?)
-        GPE_PUTSCOM(EQ_PM_L2_RCMD_DIS_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
-
-        PK_TRACE("SE8.b");
-        // Set all bits to zero prior stop core clocks
-        GPE_PUTSCOM(EQ_SCAN_REGION_TYPE, QUAD_ADDR_BASE, qloop, 0);
-
-        PK_TRACE("SE8.c");
-        // Stop L2 Clocks
-        GPE_PUTSCOM(EQ_CLK_REGION, QUAD_ADDR_BASE, qloop,
-                    (0x900000000000E000 | ((uint64_t)ex << SHIFT64(9))));
-
-        PK_TRACE("SE8.d");
+        // ------------------------------------------------------------------------
+        // EX STOP ENTRY
+        // ------------------------------------------------------------------------
 
-        // Poll for L2 clocks stopped
-        do
+        for(xentry = G_sgpe_stop_record.group.member.x_in, qloop = 0;
+            xentry > 0;
+            xentry = xentry << 2, qloop++)
         {
-            GPE_GETSCOM(EQ_CLOCK_STAT_SL, QUAD_ADDR_BASE, qloop, scom_data);
-        }
-        while(((scom_data >> SHIFT64(9)) & ex) != ex);
-
-        // MF: verify compiler generate single rlwmni
-        // MF: delay may be needed for stage latch to propagate thold
-
-        PK_TRACE("SE8: L2 Clock Stopped");
-
-        //========================
-        MARK_TRAP(SE_STOP_L2_GRID)
-        //========================
+            // if this ex is not up to entry, skip
+            if (!(ex = ((xentry & BITS16(0, 2)) >> SHIFT16(1))))
+            {
+                continue;
+            }
+
+            PK_TRACE("q[%d]exmask[%d] starts entry", qloop, ex);
+
+            //-------------------------------------------------------------------------
+            // STOP LEVEL 8
+            //-------------------------------------------------------------------------
+
+            // Update QSSR: stop_entry_ongoing
+            out32(OCB_QSSR_OR, BIT32(qloop + 20));
+
+            // Update History for ongoing stop 8 entry
+            cloop  = (ex & FST_EX_IN_QUAD) ?              0 : CORES_PER_EX;
+            climit = (ex & SND_EX_IN_QUAD) ? CORES_PER_QUAD : CORES_PER_EX;
+
+            for(; cloop < climit; cloop++)
+            {
+                SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                         CORE_ADDR_BASE,
+                                         STOP_CORE_IS_GATED,
+                                         STOP_TRANS_ENTRY,
+                                         STOP_LEVEL_8,
+                                         STOP_LEVEL_8,
+                                         STOP_REQ_DISABLE,
+                                         STOP_ACT_DISABLE);
+            }
+
+            //========================
+            MARK_TRAP(SE_STOP_L2_CLKS)
+            //========================
+
+            PK_TRACE("SE8.a");
+            // Disable L2 Snoop(quiesce L2-L3 interface, what about NCU?)
+            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_L2_RCMD_DIS_REG, qloop, ex), BIT64(0));
+            PPE_WAIT_CORE_CYCLES(loop, 256)
+
+            PK_TRACE("SE8.b");
+            // Set all bits to zero prior stop core clocks
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
+
+            PK_TRACE("SE8.c");
+            // Stop L2 Clocks
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop),
+                        (0x900000000000E000 | ((uint64_t)ex << SHIFT64(9))));
+
+            PK_TRACE("SE8.d");
+
+            // Poll for L2 clocks stopped
+            do
+            {
+                GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
+            }
+            while(((scom_data >> SHIFT64(9)) & ex) != ex);
+
+            // MF: verify compiler generate single rlwmni
+            // MF: delay may be needed for stage latch to propagate thold
+
+            PK_TRACE("SE8: L2 Clock Stopped");
+
+            //========================
+            MARK_TRAP(SE_STOP_L2_GRID)
+            //========================
+
+            PK_TRACE("SE8.e");
+            // Drop clock sync enable before switch to refclk
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_EXCGCR_CLR, qloop),
+                        (ex << SHIFT64(37)));
 
 #if !EPM_P9_TUNING
-        PK_TRACE("SE8.e");
-        // Drop clock sync enable before switch to refclk
-        GPE_PUTSCOM(EQ_QPPM_EXCGCR_CLR, QUAD_ADDR_BASE, qloop,
-                    (ex << SHIFT64(37)));
+            PK_TRACE("SE8.f");
 
-        PK_TRACE("SE8.f");
-
-        // Poll for clock sync done to drop
-        do
-        {
-            GPE_GETSCOM(QPPM_QACSR, QUAD_ADDR_BASE, qloop, scom_data);
-        }
-        while((~scom_data >> SHIFT64(37)) & ex != ex);
+            // Poll for clock sync done to drop
+            do
+            {
+                GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QACSR, qloop), scom_data);
+            }
+            while((~scom_data >> SHIFT64(37)) & ex != ex);
 
 #endif
 
-        PK_TRACE("SE8.g");
-        // Switch glsmux to refclk to save clock grid power
-        GPE_PUTSCOM(EQ_QPPM_EXCGCR_CLR, QUAD_ADDR_BASE, qloop,
-                    (ex << SHIFT64(35)));
-
-        PK_TRACE("SE8.h");
-
-        if (ex & FST_EX_IN_QUAD)
-        {
-            cloop = 0;
-            G_sgpe_stop_record.state[qloop].detail.x0act = STOP_LEVEL_8;
-            entry_ongoing[0] =
-                G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
-                STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
-        }
-        else
-        {
-            cloop = CORES_PER_EX;
-        }
-
-        if (ex & SND_EX_IN_QUAD)
-        {
-            climit = CORES_PER_QUAD;
-            G_sgpe_stop_record.state[qloop].detail.x1act = STOP_LEVEL_8;
-            entry_ongoing[1] =
-                G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
-                STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
-        }
-        else
-        {
-            climit = CORES_PER_EX;
-        }
-
-        for(; cloop < climit; cloop++)
-        {
-            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
-                                     CORE_ADDR_BASE,
-                                     STOP_CORE_IS_GATED,
-                                     entry_ongoing[cloop >> 1],
-                                     STOP_LEVEL_8,
-                                     STOP_LEVEL_8,
-                                     STOP_REQ_DISABLE,
-                                     STOP_ACT_ENABLE);
-        }
-
-        // Update QSSR: l2_stopped, drop stop_entry_ongoing
-        out32(OCB_QSSR_OR, (ex << SHIFT32((qloop << 1) + 1)) | BIT32(qloop + 20));
-
-        //=================================================
-        MARK_TAG(SE_STOP8_DONE, ((ex << 6) | (1 << qloop)))
-        //=================================================
-
-    };
+            PK_TRACE("SE8.g");
+            // Switch glsmux to refclk to save clock grid power
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_EXCGCR_CLR, qloop),
+                        (ex << SHIFT64(35)));
+
+            PK_TRACE("SE8.h");
+
+            if (ex & FST_EX_IN_QUAD)
+            {
+                cloop = 0;
+                G_sgpe_stop_record.state[qloop].detail.x0act = STOP_LEVEL_8;
+                entry_ongoing[0] =
+                    G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
+                    STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+            }
+            else
+            {
+                cloop = CORES_PER_EX;
+            }
+
+            if (ex & SND_EX_IN_QUAD)
+            {
+                climit = CORES_PER_QUAD;
+                G_sgpe_stop_record.state[qloop].detail.x1act = STOP_LEVEL_8;
+                entry_ongoing[1] =
+                    G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
+                    STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+            }
+            else
+            {
+                climit = CORES_PER_EX;
+            }
+
+            for(; cloop < climit; cloop++)
+            {
+                SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                         CORE_ADDR_BASE,
+                                         STOP_CORE_IS_GATED,
+                                         entry_ongoing[cloop >> 1],
+                                         STOP_LEVEL_8,
+                                         STOP_LEVEL_8,
+                                         STOP_REQ_DISABLE,
+                                         STOP_ACT_ENABLE);
+            }
+
+            // Update QSSR: l2_stopped, drop stop_entry_ongoing
+            out32(OCB_QSSR_OR, (ex << SHIFT32((qloop << 1) + 1)) | BIT32(qloop + 20));
+
+            //=================================================
+            MARK_TAG(SE_STOP8_DONE, ((ex << 6) | (32 >> qloop)))
+            //=================================================
+
+        };
 
     // ------------------------------------------------------------------------
     // QUAD STOP ENTRY
@@ -199,6 +207,8 @@ p9_sgpe_stop_entry()
         qentry > 0;
         qentry = qentry << 1, qloop++)
     {
+        PK_TRACE("q[%x] starts entry", qentry);
+
         // if this quad is not up to entry, skip
         if (!(qentry & BIT16(0)))
         {
@@ -211,179 +221,220 @@ p9_sgpe_stop_entry()
         out32(OCB_QSSR_OR, BIT32(qloop + 20));
 
         // Update STOP History
-        for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
-        {
-            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
-                                     CORE_ADDR_BASE,
-                                     STOP_CORE_IS_GATED,
-                                     STOP_TRANS_ENTRY,
-                                     G_sgpe_stop_record.state[qloop].detail.q_req,
-                                     STOP_LEVEL_9,
-                                     STOP_REQ_ENABLE,
-                                     STOP_ACT_DISABLE);
-        }
+        SGPE_STOP_UPDATE_HISTORY(qloop,
+                                 QUAD_ADDR_BASE,
+                                 STOP_CORE_IS_GATED,
+                                 STOP_TRANS_ENTRY,
+                                 G_sgpe_stop_record.state[qloop].detail.q_req,
+                                 STOP_LEVEL_11,
+                                 STOP_REQ_ENABLE,
+                                 STOP_ACT_DISABLE);
 
         // ------------------------------------------------------------------------
-        // STOP LEVEL 9
+        // STOP LEVEL 11.A
         //-------------------------------------------------------------------------
 
         //=================================
-        MARK_TAG(SE_PURGE_L3, (1 << qloop))
+        MARK_TAG(SE_PURGE_L3, (32 >> qloop))
         //=================================
 
-        // Assert chtm purges
-        GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE | CME_ADDR_OFFSET_EX0,
-                    qloop, BIT64(24) | BIT64(25));
-        GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE | CME_ADDR_OFFSET_EX1,
-                    qloop, BIT64(24) | BIT64(25));
+        ex = 2;
 
-        PK_TRACE("SE9.a");
+        PK_TRACE("SE11.a");
         // Disable LCO prior to purge
-        GPE_PUTSCOM(EQ_PM_LCO_DIS_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG, qloop, ex), BIT64(0));
 
-        PK_TRACE("SE9.b");
+#if !SKIP_L3_PURGE
+        PK_TRACE("SE11.b");
         // Assert Purge L3
-        GPE_PUTSCOM(PM_PURGE_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, ex), BIT64(0));
 
-        PK_TRACE("SE9.b");
+        // PK_TRACE("SE11.c");
+        // No need: Assert chtm purges
+        // todo: stop debug trace, attribute may be needed
+        //GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX0,
+        //            qloop, BIT64(24)|BIT64(25));
+        //GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX1,
+        //            qloop, BIT64(24)|BIT64(25));
+
+        PK_TRACE("SE11.d");
 
         // Poll for purge done on the same request bit thus no need to deassert
         do
         {
-            GPE_GETSCOM(PM_PURGE_REG, QUAD_ADDR_BASE, qloop, scom_data);
+#if !SKIP_L3_PURGE_ABORT
+
+            if (in32(OCB_OISR1) & BIT32(15))
+            {
+                if (in32(OCB_OPITNPRA(2)) & (BITS32((qloop << 2), 4)))
+                {
+                    for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
+                    {
+                        if (in32(OCB_OPIT2CN(((qloop << 2) + cloop))) &
+                            TYPE2_PAYLOAD_STOP_EVENT)
+                        {
+                            // Assert Purge L3 Abort
+                            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG,
+                                                         qloop, ex), BIT64(2));
+
+                            // Poll for Abort Done
+                            do
+                            {
+                                GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG,
+                                                             qloop, ex), scom_data);
+                            }
+                            while(scom_data & (BIT64(0) | BIT64(2)));
+
+                            // Deassert LCO Disable
+                            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG,
+                                                         qloop, ex), 0);
+                            // Notify PGPE to resume
+                            l3_purge_aborted = 1;
+                            break;
+                        }
+                    }
+                }
+            }
+
+#endif
+            GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, ex), scom_data);
         }
         while(scom_data & BIT64(0));
 
+        if (l3_purge_aborted)
+        {
+            continue;
+        }
+
+#endif
+
         //====================
         MARK_TRAP(SE_PURGE_PB)
         //====================
 
-        PK_TRACE("SE9.b");
-        // Purge PowerBus
-        GPE_PUTSCOM(EQ_QPPM_QCCR_WOR, QUAD_ADDR_BASE, qloop, BIT64(30));
+#if EPM_P9_TUNING
+        // PK_TRACE("SE11.e");
+        // No need: Poll for chtm purge done
+        // do {
+        //     GPE_GETSCOM(CME_SCOM_EISR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX0,
+        //                 qloop, scom_data);
+        // } while((scom_data & BITS64(24,2)) != BITS64(22,2));
+        // do {
+        //     GPE_GETSCOM(CME_SCOM_EISR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX1,
+        //                 qloop, scom_data);
+        // } while((scom_data & BITS64(24,2)) != BITS64(22,2));
+        // cme scom eisr is read-only, cannot clear the eisr here, clear in cme
+#endif
+
+        // PK_TRACE("SE11.f");
+        // No need: Deassert chtm purges
+        //GPE_PUTSCOM(CME_SCOM_SICR_CLR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX0,
+        //            qloop, BIT64(24)|BIT64(25));
+        //GPE_PUTSCOM(CME_SCOM_SICR_CLR, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX1,
+        //            qloop, BIT64(24)|BIT64(25));
+
+        PK_TRACE("SE11.g");
+        // Assert PowerBus purge
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, qloop), BIT64(30));
 
-        PK_TRACE("SE9.b");
+        PK_TRACE("SE11.h");
 
-        // Poll purge PowerBus done
+        // Poll PowerBus purge done
         do
         {
-            GPE_GETSCOM(EQ_QPPM_QCCR, QUAD_ADDR_BASE, qloop, scom_data);
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR, qloop), scom_data);
         }
         while(!(scom_data & BIT64(31)));
 
+        PK_TRACE("SE11.i");
+        // Deassert PowerBus purge
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(30));
+
         //==========================================
-        MARK_TAG(SE_WAIT_PGPE_SUSPEND, (1 << qloop))
+        MARK_TAG(SE_WAIT_PGPE_SUSPEND, (32 >> qloop))
         //==========================================
-        // Poll PGPE Suspend Ack
+        // TODO: Poll PGPE Suspend Ack
 
         //=====================================
-        MARK_TAG(SE_STOP_L3_CLKS, (1 << qloop))
+        MARK_TAG(SE_STOP_CACHE_CLKS, (32 >> qloop))
         //=====================================
 
-        PK_TRACE("SE9.d");
+        PK_TRACE("SE11.j");
         // Assert refresh quiesce prior to L3 (refresh domain) stop clk
         // Edram quiesce is asserted by hardware when l3 thold is asserted in cc
-        GPE_PUTSCOM(DRAM_REF_REG, qloop, QUAD_ADDR_BASE, BIT64(7));
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_DRAM_REF_REG, qloop, ex), scom_data);
+        scom_data |= BIT64(7);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_DRAM_REF_REG, qloop, ex), scom_data);
+
+        // todo: check NCU_SATUS_REG[0:3] for all zeros
 
-        PK_TRACE("SE9.c");
+        PK_TRACE("SE11.k");
         // Raise Cache Logical fence
-        GPE_PUTSCOM(EQ_NET_CTRL0_WOR, QUAD_ADDR_BASE, qloop, BIT64(18));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(18));
 
         // clock off cache chiplet
-        PK_TRACE("SE9.e");
+        PK_TRACE("SE11.l");
         // Set all bits to zero prior stop core clocks
-        GPE_PUTSCOM(EQ_SCAN_REGION_TYPE, QUAD_ADDR_BASE, qloop, 0);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
 
-        PK_TRACE("SE9.f");
+        PK_TRACE("SE11.m");
         // Stop Cache Clocks
-        GPE_PUTSCOM(EQ_CLK_REGION, QUAD_ADDR_BASE, qloop, 0x9FFC00000000E000);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop), 0x9F3E00000000E000);
 
-        PK_TRACE("SE9.g");
+        PK_TRACE("SE11.n");
 
         // Poll for Cache clocks stopped
         do
         {
-            GPE_GETSCOM(EQ_CLOCK_STAT_SL, QUAD_ADDR_BASE, qloop, scom_data);
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
         }
         while((scom_data & BITS64(4, 10)) != BITS64(4, 10));
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
 
-        PK_TRACE("SE9: Cache Clock Stopped");
+        PK_TRACE("SE11: Cache Clock Stopped");
 
-        PK_TRACE("SE9.h");
+        PK_TRACE("SE11.o");
         // Switch glsmux to refclk to save clock grid power
-        GPE_PUTSCOM(PPM_CGCR_CLR, QUAD_ADDR_BASE, qloop, BIT64(3));
-
-        PK_TRACE("SE9.i");
-        // Update Stop History
-        G_sgpe_stop_record.state[qloop].detail.q_act = STOP_LEVEL_9;
-        entry_ongoing[0] =
-            G_sgpe_stop_record.state[qloop].detail.q_req == STOP_LEVEL_9 ?
-            STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
-        /*
-        for(cloop=0;cloop<CORES_PER_QUAD;cloop++) {
-            SGPE_STOP_UPDATE_HISTORY(((qloop<<2)+cloop),
-                                     CORE_ADDR_BASE,
-                                     STOP_CORE_IS_GATED,
-                                     entry_ongoing[0],
-                                     STOP_LEVEL_9,
-                                     STOP_LEVEL_9,
-                                     STOP_REQ_DISABLE,
-                                     STOP_ACT_ENABLE);
-        }*/
-        SGPE_STOP_UPDATE_HISTORY(qloop,
-                                 QUAD_ADDR_BASE,
-                                 STOP_CORE_IS_GATED,
-                                 entry_ongoing[0],
-                                 STOP_LEVEL_9,
-                                 STOP_LEVEL_9,
-                                 STOP_REQ_DISABLE,
-                                 STOP_ACT_ENABLE);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), BIT64(3));
 
         // Update QSSR: quad_stopped
         out32(OCB_QSSR_OR, BIT32(qloop + 14));
 
         //===================================
-        MARK_TAG(SE_STOP9_DONE, (1 << qloop))
+        MARK_TAG(SE_STOP_CACHE_CLKS_DONE, (32 >> qloop))
         //===================================
 
-        if (!entry_ongoing[0])
-        {
-            // Update QSSR: drop stop_entry_ongoing
-            out32(OCB_QSSR_CLR, BIT32(qloop + 20));
-            continue;
-        }
-
         //-------------------------------------------------------------------------
-        // STOP LEVEL 11
+        // STOP LEVEL 11.B
         //-------------------------------------------------------------------------
 
         //========================================
-        MARK_TAG(SE_POWER_OFF_CACHE, (1 << qloop))
+        MARK_TAG(SE_POWER_OFF_CACHE, (32 >> qloop))
         //========================================
 
         // Assert Cache Electrical Fence
-        PK_TRACE("SE11.a");
-        GPE_PUTSCOM(EQ_NET_CTRL0_WOR, QUAD_ADDR_BASE, qloop, BIT64(26));
+        PK_TRACE("SE11.q");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(26));
 
+#if !STOP_PRIME
         // L3 edram shutdown
-        PK_TRACE("SE11.b");
-        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
-                    BIT64(3) | BIT64(7));
-        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
-                    BIT64(2) | BIT64(6));
-        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
-                    BIT64(1) | BIT64(5));
-        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
-                    BIT64(0) | BIT64(4));
+        PK_TRACE("SE11.r");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(7));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(6));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(5));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(4));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(3));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(2));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(1));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(0));
 
         // Make sure we are not forcing PFET for VDD or VCS off
         // vdd_pfet_force_state == 00 (Nop)
         // vcs_pfet_force_state == 00 (Nop)
-        PK_TRACE("SE11.c");
-        GPE_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+        PK_TRACE("SE11.s");
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS, qloop), scom_data);
 
         if (scom_data & BITS64(0, 4))
         {
@@ -394,31 +445,31 @@ p9_sgpe_stop_entry()
         // vdd_pfet_val/sel_override     = 0 (disbaled)
         // vcs_pfet_val/sel_override     = 0 (disbaled)
         // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
-        PK_TRACE("SE11.d");
-        GPE_PUTSCOM(PPM_PFCS_CLR, QUAD_ADDR_BASE, qloop, BITS64(4, 4) | BIT64(8));
+        PK_TRACE("SE11.t");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, qloop), BITS64(4, 4) | BIT64(8));
 
         // Power Off Core VDD
         // vdd_pfet_force_state = 01 (Force Voff)
         // vcs_pfet_force_state = 01 (Force Voff)
-        PK_TRACE("SE11.e");
-        GPE_PUTSCOM(PPM_PFCS_OR, QUAD_ADDR_BASE, qloop, BIT64(1) | BIT64(3));
+        PK_TRACE("SE11.u");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, qloop), BIT64(1) | BIT64(3));
 
         // Poll for power gate sequencer state: 0x8 (FSM Idle)
-        PK_TRACE("SE11.f");
+        PK_TRACE("SE11.v");
 
         do
         {
-            GPE_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS, qloop), scom_data);
         }
         while(!(scom_data & (BIT64(42) | BIT64(50))));
 
 #if !EPM_P9_TUNING
         // Optional: Poll for vdd_pg_sel/vcs_pg_sel being: 0x8
-        PK_TRACE("SE11.g");
+        PK_TRACE("SE11.w");
 
         do
         {
-            CME_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+            CME_GETSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS, qloop), scom_data);
         }
         while(!(scom_data & BIT64(46) | BIT64(54)));
 
@@ -426,10 +477,11 @@ p9_sgpe_stop_entry()
 
         // Turn Off Force Voff
         // vdd_pfet_force_state = 00 (Nop)
-        PK_TRACE("SE11.h");
-        GPE_PUTSCOM(PPM_PFCS_CLR, QUAD_ADDR_BASE, qloop, BITS64(0, 4));
+        PK_TRACE("SE11.x");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, qloop), BITS64(0, 4));
+#endif
 
-        PK_TRACE("SE11.i");
+        PK_TRACE("SE11.y");
         G_sgpe_stop_record.state[qloop].detail.q_act = STOP_LEVEL_11;
 
         for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
@@ -457,13 +509,12 @@ p9_sgpe_stop_entry()
         out32(OCB_QSSR_CLR, BIT32(qloop + 20));
 
         //====================================
-        MARK_TAG(SE_STOP11_DONE, (1 << qloop))
+        MARK_TAG(SE_STOP11_DONE, (32 >> qloop))
         //====================================
-    };
+    }
 
     // Enable Type2 Interrupt
     out32(OCB_OIMR1_CLR, BIT32(15));
-
     //============================
     MARK_TRAP(ENDSCOPE_STOP_ENTRY)
     //============================