CME/SGPE: STOP11 CME/SGPE Images Snapshot

Change-Id: I13d6c747e7b4b0b19e317ba7a5b74f852dee6ddf
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/23571
Tested-by: Jenkins Server
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

17 files changed, 217 insertions, 71 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
index 4a0d0968..6d8636dc 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
@@ -72,6 +72,7 @@
 #define C_CPLT_CTRL0_CLEAR 0x20000020
 #define C_CPLT_CTRL1_OR    0x20000011
 #define C_CPLT_CTRL1_CLEAR 0x20000021
+#define C_CPLT_STAT0       0x20000100
 
 #define PERV_CPLT_CTRL0_OR 0x00000010
 #define PERV_CPLT_CTRL0_CLEAR 0x00000020
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index 6bf5afdf..a41a8edf 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -234,6 +234,7 @@ p9_cme_stop_entry()
 
         PK_TRACE("SE2.j");
         // Raise Core Chiplet Fence
+        CME_PUTSCOM(C_CPLT_CTRL0_OR, core, BIT64(2));
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(18));
 
         PK_TRACE("SE2.k");
@@ -280,10 +281,14 @@ p9_cme_stop_entry()
 
         PK_TRACE("SE2.p");
         // Switch glsmux to refclk to save clock grid power
-        CME_PUTSCOM(C_PPM_CGCR, core, BIT64(3));
+        CME_PUTSCOM(C_PPM_CGCR, core, 0);
 
         // Assert PCB Fence
-        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
+        //CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
+        // Assert Vital Fence
+        CME_PUTSCOM(C_CPLT_CTRL1_OR,  core, BIT64(3));
+        // Assert Regional Fences
+        CME_PUTSCOM(C_CPLT_CTRL1_OR, core, 0xFFFF700000000000);
 
         PK_TRACE("SE2.q");
         // Update Stop History: In Core Stop Level 2
@@ -500,11 +505,14 @@ p9_cme_stop_entry()
         MARK_TAG(SE_POWER_OFF_CORE, core)
         //===============================
 
-#if !STOP_PRIME
-        // Assert Cores Electrical Fences
+        // DD: Assert Cores Vital Thold/PCB Fence/Electrical Fence
         PK_TRACE("SE4.a");
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(26));
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(16));
 
+#if !STOP_PRIME
+#if !EPM_P9_TUNING
         // Make sure we are not forcing PFET for VDD off
         // vdd_pfet_force_state == 00 (Nop)
         PK_TRACE("SE4.b");
@@ -515,6 +523,8 @@ p9_cme_stop_entry()
             return CME_STOP_ENTRY_VDD_PFET_NOT_IDLE;
         }
 
+#endif
+
         // Prepare PFET Controls
         // vdd_pfet_val/sel_override     = 0 (disbaled)
         // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
@@ -546,7 +556,6 @@ p9_cme_stop_entry()
         while(!(scom_data & BIT64(46)));
 
 #endif
-
         // Turn Off Force Voff
         // vdd_pfet_force_state = 00 (Nop)
         PK_TRACE("SE4.g");
@@ -695,7 +704,7 @@ p9_cme_stop_entry()
             //===========================
             do
             {
-#if !SKIP_ABORT
+#if !SKIP_L2_PURGE_ABORT
 
                 if(in32(CME_LCL_EINR) & BITS32(12, 6))
                 {
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
index 49d34942..c86574ac 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
@@ -159,7 +159,6 @@ p9_cme_stop_exit()
         do   //catchup loop
         {
 
-#if !STOP_PRIME
             // todo PK_TRACE("BCE Runtime Kickoff");
 
             // todo for catch up case
@@ -176,6 +175,7 @@ p9_cme_stop_exit()
             p9_hcd_core_chiplet_reset(core);
             MARK_TRAP(SX_CHIPLET_RESET_END)
 
+#if !STOP_PRIME
 #if !SKIP_CATCHUP
 
             //catchup
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
index 732b6a26..e929ff31 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
@@ -69,6 +69,6 @@ p9_cme_stop_doorbell_handler(void* arg, PkIrqId irq)
         out32(CME_LCL_EIMR_CLR, BIT32(13) | BIT32(15) | BIT32(17));
     }
 
-    // TODO mask pc_itr_pending as workaround for double interrupts of pc and rwu
+    //TODO mask pc_itr_pending as workaround for double interrupts of pc and rwu
     out32(CME_LCL_EIMR_OR, BITS32(12, 2));
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
index c4fdc9b1..ad044dfb 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
@@ -35,6 +35,15 @@ p9_hcd_core_chiplet_reset(uint32_t core)
     PK_TRACE("Init NETWORK_CONTROL0, step needed for hotplug");
     CME_PUTSCOM(CPPM_NC0INDIR_OR, core, NET_CTRL0_INIT_VECTOR);
 
+    PK_TRACE("Assert Core Progdly and DCC Bypass");
+    CME_PUTSCOM(CPPM_NC1INDIR_OR, core, BITS64(1, 2));
+
+    PK_TRACE("Assert Core DCC Reset");
+    CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(2));
+
+    PK_TRACE("Assert Vital Thold");
+    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(16));
+
     PK_TRACE("ONLY till TP030: SET VITL_PHASE=1");
     CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(8));
     PPE_WAIT_CORE_CYCLES(loop, 50);
@@ -50,6 +59,7 @@ p9_hcd_core_chiplet_reset(uint32_t core)
 
     PK_TRACE("Remove chiplet electrical fence via NET_CTRL0[26]");
     CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(26));
+    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(25));
 
     PK_TRACE("ONLY till TP030: SET SYNC_PULSE_DELAY=0b0011");
     CME_GETSCOM(C_SYNC_CONFIG, core, CME_SCOM_AND, data);
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
index 1bc415b2..7dce5e04 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
@@ -30,11 +30,11 @@ int
 p9_hcd_core_poweron(uint32_t core)
 {
     int rc = CME_STOP_SUCCESS;
-    uint64_t scom_data;
 
     PK_TRACE("Set core glsmux reset");
     CME_PUTSCOM(C_PPM_CGCR, core, BIT64(0));
-
+#if !STOP_PRIME
+    uint64_t scom_data;
 #if !EPM_P9_TUNNING
     // vdd_pfet_force_state == 00 (Nop)
     PK_TRACE("Make sure we are not forcing PFET for VDD off");
@@ -45,11 +45,12 @@ p9_hcd_core_poweron(uint32_t core)
         return CME_STOP_ENTRY_VDD_PFET_NOT_IDLE;
     }
 
+#endif
+
     // vdd_pfet_val/sel_override     = 0 (disbaled)
     // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
     PK_TRACE("Prepare PFET Controls");
     CME_PUTSCOM(PPM_PFCS_CLR, core, BIT64(4) | BIT64(5) | BIT64(8));
-#endif
 
     // vdd_pfet_force_state = 11 (Force Von)
     PK_TRACE("Power Off Core VDD");
@@ -75,7 +76,7 @@ p9_hcd_core_poweron(uint32_t core)
     while(scom_data & BIT64(46));
 
     MARK_TRAP(SX_POWERON_PG_SEL)
-
+#endif
     // vdd_pfet_force_state = 00 (Nop)
     PK_TRACE("Turn Off Force Von");
     CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
index 5ef8c995..5f383b74 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
@@ -34,7 +34,7 @@ p9_hcd_core_startclocks(uint32_t core)
 
     PK_TRACE("Setup OPCG_ALIGN Register");
     CME_GETSCOM(C_OPCG_ALIGN, core, CME_SCOM_AND, data);
-    data = data & (BITS64(0, 4) & BITS64(12, 8) & BITS64(52, 12));
+    data = data & ~(BITS64(0, 4) & BITS64(12, 8) & BITS64(52, 12));
     data = data | (BIT64(1) | BIT64(3) | BIT64(59));
     CME_PUTSCOM(C_OPCG_ALIGN, core, data);
 
@@ -44,9 +44,25 @@ p9_hcd_core_startclocks(uint32_t core)
     PK_TRACE("Drop vital fences via CPLT_CTRL1");
     CME_PUTSCOM(C_CPLT_CTRL1_CLEAR, core, BIT64(3));
 
+    PK_TRACE("Raise core clock sync enable");
+    CME_PUTSCOM(CPPM_CACCR_OR, core, BIT64(15));
+
+#if !EPM_P9_TUNING
+    PK_TRACE("Poll for core clock sync done to raise");
+
+    do
+    {
+        CME_GETSCOM(CPPM_CACSR, core, CME_SCOM_AND, data);
+    }
+    while(~data & BIT64(13));
+
+#endif
+
     PK_TRACE("Reset abstclk & syncclk muxsel(io_clk_sel) via CPLT_CTRL0[0:1]");
     CME_PUTSCOM(C_CPLT_CTRL0_CLEAR, core, BITS64(0, 2));
 
+    // align_chiplets()
+
     PK_TRACE("Set flushmode_inhibit via CPLT_CTRL0[2]");
     CME_PUTSCOM(C_CPLT_CTRL0_OR, core, BIT64(2));
 
@@ -57,28 +73,22 @@ p9_hcd_core_startclocks(uint32_t core)
     CME_GETSCOM(C_SYNC_CONFIG, core, CME_SCOM_AND, data);
     data = data | BIT64(7);
     CME_PUTSCOM(C_SYNC_CONFIG, core, data);
-    data = data & BIT64(7);
+    data = data & ~BIT64(7);
     CME_PUTSCOM(C_SYNC_CONFIG, core, data);
 
-    // align chiplets
-
-    PK_TRACE("Clear force_align via CPLT_CTRL0[3]");
-    CME_PUTSCOM(C_CPLT_CTRL0_CLEAR, core, BIT64(3));
-    PPE_WAIT_CORE_CYCLES(loop, 450);
-
-    PK_TRACE("Raise core clock sync enable");
-    CME_PUTSCOM(CPPM_CACCR_OR, core, BIT64(15));
-
-#if !EPM_P9_TUNING
-    PK_TRACE("Poll for core clock sync done to raise");
+    PK_TRACE("Check chiplet_is_aligned");
 
     do
     {
-        CME_GETSCOM(CPPM_CACSR, core, CME_SCOM_AND, data);
+        CME_GETSCOM(C_CPLT_STAT0, core, CME_SCOM_AND, data);
     }
-    while(~data & BIT64(13));
+    while(~data & BIT64(9));
 
-#endif
+    PK_TRACE("Clear force_align via CPLT_CTRL0[3]");
+    CME_PUTSCOM(C_CPLT_CTRL0_CLEAR, core, BIT64(3));
+    PPE_WAIT_CORE_CYCLES(loop, 450);
+
+    // clock_start()
 
     PK_TRACE("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
     CME_PUTSCOM(C_SCAN_REGION_TYPE, core, 0);
@@ -106,8 +116,8 @@ p9_hcd_core_startclocks(uint32_t core)
     PK_TRACE("Drop chiplet fence via NC0INDIR[18]");
     CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(18));
 
-    PK_TRACE("Drop fence to allow PCB operations to chiplet via NC0INDIR[26]");
-    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(25));
+    //PK_TRACE("Drop fence to allow PCB operations to chiplet via NC0INDIR[26]");
+    //CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(25));
 
     // checkstop
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
index a942ea41..ccad9c57 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
@@ -34,6 +34,18 @@ p9_hcd_cache_chiplet_reset(uint8_t quad)
     PK_TRACE("Init NETWORK_CONTROL0, step needed for hotplug");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), NET_CTRL0_INIT_VECTOR);
 
+    PK_TRACE("Assert L3, L2-0, L2-1 Progdly, DCC Bypass and Reset");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL1_WOR, quad), (BITS64(1, 2) | BITS64(23, 2)));
+
+    PK_TRACE("Assert Skew Adjust Reset");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), BIT64(2));
+
+    PK_TRACE("Set DPLL ff_bypass to 1");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_DPLL_CTRL_OR, quad), BIT64(2));
+
+    PK_TRACE("Drop Vital Thold");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(16));
+
     MARK_TRAP(SX_CHIPLET_RESET_GLSMUX_RESET)
 
     PK_TRACE("Init Cache Glitchless Mux Reset/Select via CLOCK_GRID_CTRL[0:3]");
@@ -47,6 +59,10 @@ p9_hcd_cache_chiplet_reset(uint8_t quad)
 
     PK_TRACE("Remove chiplet electrical fence via NET_CTRL0[26]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(26));
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(25));
+
+    PK_TRACE("Remove PCB fence via NET_CTRL0[25]");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(25));
 
 #if !SKIP_SCAN0
     // Marker for scan0
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_dpll_setup.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_dpll_setup.c
index 24e35bf4..89da6490 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_dpll_setup.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_dpll_setup.c
@@ -53,6 +53,9 @@ p9_hcd_cache_dpll_setup(uint8_t quad)
     PK_TRACE("Drop DPLL Test Mode and Reset");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BITS64(3, 2));
 
+    PK_TRACE("Drop DPLL Clock Region Fence");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_CLEAR, quad), BIT64(14));
+
     PK_TRACE("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, quad), 0);
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_l2_startclocks.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_l2_startclocks.c
index bbc3909e..3a929cdc 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_l2_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_l2_startclocks.c
@@ -41,12 +41,33 @@ p9_hcd_cache_l2_startclocks(uint8_t ex, uint8_t quad)
     scom_data = scom_data | (BIT64(1) | BIT64(3) | BIT64(59));
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_OPCG_ALIGN, quad), scom_data);
 
+    PK_TRACE("Drop L2 Regional Fences");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_CLEAR, quad),
+                ((uint64_t)ex << SHIFT64(9)));
+
+    // align_chiplets()
+
     PK_TRACE("Set flushmode_inhibit via CPLT_CTRL0[2]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_OR, quad), BIT64(2));
 
     PK_TRACE("Set force_align via CPLT_CTRL0[3]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_OR, quad), BIT64(3));
 
+    PK_TRACE("Set/Unset clear_chiplet_is_aligned via SYNC_CONFIG[7]");
+    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), scom_data);
+    scom_data = scom_data | BIT64(7);
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), scom_data);
+    scom_data = scom_data & ~BIT64(7);
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), scom_data);
+
+    PK_TRACE("Check chiplet_is_aligned");
+
+    do
+    {
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_STAT0, quad), scom_data);
+    }
+    while(~scom_data & BIT64(9));
+
     PK_TRACE("Clear force_align via CPLT_CTRL0[3]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_CLEAR, quad), BIT64(3));
 
@@ -64,6 +85,10 @@ p9_hcd_cache_l2_startclocks(uint8_t ex, uint8_t quad)
 
 #endif
 
+    // -------------------------------
+    // Start L2 Clock
+    // -------------------------------
+
     PK_TRACE("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, quad), 0);
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_poweron.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_poweron.c
index f3a60650..8427eaf6 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_poweron.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_poweron.c
@@ -30,7 +30,6 @@ int
 p9_hcd_cache_poweron(uint8_t quad)
 {
     int rc = SGPE_STOP_SUCCESS;
-    uint64_t scom_data;
 
     PK_TRACE("Set L3 glsmux reset via CLOCK_GRID_CTRL[0]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, quad), BIT64(0));
@@ -41,6 +40,8 @@ p9_hcd_cache_poweron(uint8_t quad)
     PK_TRACE("Set DPLL ff_bypass via EQ_QPPM_DPLL_CTRL[2]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_DPLL_CTRL_OR, quad), BIT64(2));
 
+#if !STOP_PRIME
+    uint64_t scom_data;
 #if !EPM_P9_TUNNING
     // vdd_pfet_force_state == 00 (Nop)
     PK_TRACE("Make sure we are not forcing PFET for VDD off");
@@ -51,15 +52,17 @@ p9_hcd_cache_poweron(uint8_t quad)
         return SGPE_STOP_ENTRY_VDD_PFET_NOT_IDLE;
     }
 
+#endif
+
     // vdd_pfet_val/sel_override     = 0 (disbaled)
+    // vcs_pfet_val/sel_override     = 0 (disbaled)
     // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
     PK_TRACE("Prepare PFET Controls");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, quad),
-                BIT64(4) | BIT64(5) | BIT64(8));
-#endif
+                BITS64(4, 4) | BIT64(8));
 
     // vdd_pfet_force_state = 11 (Force Von)
-    PK_TRACE("Power Off Core VDD");
+    PK_TRACE("Power Off Cache VDD");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, quad), BITS64(0, 2));
 
     PK_TRACE("Poll for power gate sequencer state: 0x8 (FSM Idle)");
@@ -70,6 +73,18 @@ p9_hcd_cache_poweron(uint8_t quad)
     }
     while(!(scom_data & BIT64(42)));
 
+    // vcs_pfet_force_state = 11 (Force Von)
+    PK_TRACE("Power Off Cache VCS");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, quad), BITS64(2, 2));
+
+    PK_TRACE("Poll for power gate sequencer state: 0x8 (FSM Idle)");
+
+    do
+    {
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS, quad), scom_data);
+    }
+    while(!(scom_data & BIT64(50)));
+
     MARK_TRAP(SX_POWERON_DONE)
 
 #if !EPM_P9_TUNNING
@@ -82,11 +97,11 @@ p9_hcd_cache_poweron(uint8_t quad)
     while(scom_data & BIT64(46));
 
     MARK_TRAP(SX_POWERON_PG_SEL)
+#endif
 
     // vdd_pfet_force_state = 00 (Nop)
     PK_TRACE("Turn Off Force Von");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, quad), BITS64(0, 2));
 #endif
-
     return rc;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
index 5cf05ee3..00d5e21e 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
@@ -37,7 +37,8 @@ p9_hcd_cache_startclocks(uint8_t quad)
     // -------------------------------
 
     PK_TRACE("Enable L3 EDRAM/LCO setup on both EXs");
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), BIT64(23) | BIT64(24));
+    //DD:GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), BIT64(23) | BIT64(24));
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), BIT64(23));
 
     // 0x0 -> 0x8 -> 0xC -> 0xE -> 0xF to turn on edram
     // stagger EDRAM turn-on per EX (not both at same time)
@@ -49,18 +50,19 @@ p9_hcd_cache_startclocks(uint8_t quad)
     PPE_WAIT_CORE_CYCLES(loop, 100);
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(3));
     PPE_WAIT_CORE_CYCLES(loop, 100);
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(4));
-    PPE_WAIT_CORE_CYCLES(loop, 100);
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(5));
-    PPE_WAIT_CORE_CYCLES(loop, 100);
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(6));
-    PPE_WAIT_CORE_CYCLES(loop, 100);
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(7));
-    PPE_WAIT_CORE_CYCLES(loop, 100);
-
+    /* DD:
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(4));
+        PPE_WAIT_CORE_CYCLES(loop, 100);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(5));
+        PPE_WAIT_CORE_CYCLES(loop, 100);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(6));
+        PPE_WAIT_CORE_CYCLES(loop, 100);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, quad), BIT64(7));
+        PPE_WAIT_CORE_CYCLES(loop, 100);
+    */
     PK_TRACE("Setup OPCG_ALIGN Register");
     GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_OPCG_ALIGN, quad), data);
-    data = data & (BITS64(0, 4) & BITS64(12, 8) & BITS64(52, 12));
+    data = data & ~(BITS64(0, 4) & BITS64(12, 8) & BITS64(52, 12));
     data = data | (BIT64(1) | BIT64(3) | BIT64(59));
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_OPCG_ALIGN, quad), data);
 
@@ -68,7 +70,7 @@ p9_hcd_cache_startclocks(uint8_t quad)
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_CLEAR, quad),
                 0xFFFF700000000000);
 
-    PK_TRACE("Drop vital fence via CPLT_CTRL1[4]");
+    PK_TRACE("Drop vital fence via CPLT_CTRL1[3]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_CLEAR, quad), BIT64(3));
 
     PK_TRACE("Reset abstclk & syncclk muxsel(io_clk_sel) via CPLT_CTRL0[0:1]");
@@ -76,13 +78,28 @@ p9_hcd_cache_startclocks(uint8_t quad)
 
     /// @todo set fabric node/chip ID values(read from nest chiplet) still need?
 
+    // align_chiplets()
+
     PK_TRACE("Set flushmode_inhibit via CPLT_CTRL0[2]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_OR, quad), BIT64(2));
 
     PK_TRACE("Set force_align via CPLT_CTRL0[3]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_OR, quad), BIT64(3));
 
-    /// align chiplets
+    PK_TRACE("Set/Unset clear_chiplet_is_aligned via SYNC_CONFIG[7]");
+    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), data);
+    data = data | BIT64(7);
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), data);
+    data = data & ~BIT64(7);
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SYNC_CONFIG, quad), data);
+
+    PK_TRACE("Check chiplet_is_aligned");
+
+    do
+    {
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_STAT0, quad), data);
+    }
+    while(~data & BIT64(9));
 
     PK_TRACE("Clear force_align via CPLT_CTRL0[3]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_CLEAR, quad), BIT64(3));
@@ -95,11 +112,13 @@ p9_hcd_cache_startclocks(uint8_t quad)
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, quad), 0);
 
     PK_TRACE("Start clock(arrays+nsl clock region) via CLK_REGION");
-    data = 0x5F3C000000006000;
+    //DD: data = 0x5F3C000000006000;
+    data = 0x5E3C000000006000;
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, quad), data);
 
     PK_TRACE("Start clock(sl+refresh clock region) via CLK_REGION");
-    data = 0x5F3C00000000E000;
+    //DD: data = 0x5F3C00000000E000;
+    data = 0x5E3C00000000E000;
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, quad), data);
 
     // Read Clock Status Register (Cache chiplet)
@@ -110,7 +129,9 @@ p9_hcd_cache_startclocks(uint8_t quad)
     {
         GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, quad), data);
     }
-    while((~data & (BITS64(4, 4) | BITS64(10, 4))) != (BITS64(4, 4) | BITS64(10, 4)));
+
+    //while((~data & (BITS64(4, 3) | BITS64(10, 4))) != (BITS64(4, 3) | BITS64(10, 4)));
+    while((~data & (BITS64(4, 3) | BITS64(10, 4))) != (BITS64(4, 3) | BITS64(10, 4)));
 
     PK_TRACE("L3 clock running now");
 
@@ -124,8 +145,8 @@ p9_hcd_cache_startclocks(uint8_t quad)
     PK_TRACE("Drop chiplet fence via NET_CTRL0[18]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(18));
 
-    PK_TRACE("Drop fence to allow PCB operations to chiplet via NET_CTRL0[25]");
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(25));
+    //PK_TRACE("Drop fence to allow PCB operations to chiplet via NET_CTRL0[25]");
+    //GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(25));
 
     /// @todo Check the Global Checkstop FIR of dedicated EX chiplet
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 43373e80..e2a644c9 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -49,6 +49,7 @@
 
 #include "p9_stop_common.h"
 
+#define EQ_SYNC_CONFIG           0x10030000
 #define EQ_OPCG_ALIGN            0x10030001
 #define EQ_SCAN_REGION_TYPE      0x10030005
 #define EQ_CLK_REGION            0x10030006
@@ -57,11 +58,14 @@
 #define EQ_BIST                  0x100F000B
 #define EQ_NET_CTRL0_WAND        0x100F0041
 #define EQ_NET_CTRL0_WOR         0x100F0042
+#define EQ_NET_CTRL1_WAND        0x100F0045
+#define EQ_NET_CTRL1_WOR         0x100F0046
 
 #define EQ_CPLT_CTRL0_OR         0x10000010
 #define EQ_CPLT_CTRL0_CLEAR      0x10000020
 #define EQ_CPLT_CTRL1_OR         0x10000011
 #define EQ_CPLT_CTRL1_CLEAR      0x10000021
+#define EQ_CPLT_STAT0            0x10000100
 
 #define EQ_QPPM_DPLL_CTRL_CLEAR  0x100F0153
 #define EQ_QPPM_DPLL_CTRL_OR     0x100F0154
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index b3586d13..ba3453f7 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -97,6 +97,9 @@ p9_sgpe_stop_entry()
             //========================
 
             PK_TRACE("SE8.a");
+            // Assert L2 Regional Fences
+            GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop),
+                        ((uint64_t)ex << SHIFT64(9)));
             // Disable L2 Snoop(quiesce L2-L3 interface, what about NCU?)
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_L2_RCMD_DIS_REG, qloop, ex), BIT64(0));
             PPE_WAIT_CORE_CYCLES(loop, 256)
@@ -273,6 +276,7 @@ p9_sgpe_stop_entry()
                         if (in32(OCB_OPIT2CN(((qloop << 2) + cloop))) &
                             TYPE2_PAYLOAD_STOP_EVENT)
                         {
+                            MARK_TRAP(SE_PURGE_L3_ABORT)
                             // Assert Purge L3 Abort
                             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG,
                                                          qloop, ex), BIT64(2));
@@ -285,6 +289,8 @@ p9_sgpe_stop_entry()
                             }
                             while(scom_data & (BIT64(0) | BIT64(2)));
 
+                            MARK_TRAP(SE_PURGE_L3_ABORT_DONE)
+
                             // Deassert LCO Disable
                             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG,
                                                          qloop, ex), 0);
@@ -369,6 +375,8 @@ p9_sgpe_stop_entry()
         // todo: check NCU_SATUS_REG[0:3] for all zeros
 
         PK_TRACE("SE11.k");
+        // Assert flush_inhibit
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL0_OR, qloop), BIT64(2));
         // Raise Cache Logical fence
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(18));
 
@@ -379,7 +387,8 @@ p9_sgpe_stop_entry()
 
         PK_TRACE("SE11.m");
         // Stop Cache Clocks
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop), 0x9F3E00000000E000);
+        //GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop), 0x9F3E00000000E000);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop), 0x9E3E00000000E000);
 
         PK_TRACE("SE11.n");
 
@@ -388,7 +397,9 @@ p9_sgpe_stop_entry()
         {
             GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
         }
-        while((scom_data & BITS64(4, 10)) != BITS64(4, 10));
+
+        //while((scom_data & (BITS64(4, 4) | BITS64(10, 4))) != (BITS64(4, 4) | BITS64(10, 4)));
+        while((scom_data & (BITS64(4, 3) | BITS64(10, 5))) != (BITS64(4, 3) | BITS64(10, 5)));
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
@@ -397,7 +408,12 @@ p9_sgpe_stop_entry()
 
         PK_TRACE("SE11.o");
         // Switch glsmux to refclk to save clock grid power
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), BIT64(3));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), 0);
+        // Assert Vital Fence
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop), BIT64(3));
+        // Raise Partial Good Fences
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop),
+                    0xFFFF700000000000);
 
         // Update QSSR: quad_stopped
         out32(OCB_QSSR_OR, BIT32(qloop + 14));
@@ -414,22 +430,27 @@ p9_sgpe_stop_entry()
         MARK_TAG(SE_POWER_OFF_CACHE, (32 >> qloop))
         //========================================
 
-        // Assert Cache Electrical Fence
+        // DD: Assert Cache Vital Thold/PCB Fence/Electrical Fence
         PK_TRACE("SE11.q");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(25));
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(26));
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(16));
 
-#if !STOP_PRIME
         // L3 edram shutdown
         PK_TRACE("SE11.r");
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(7));
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(6));
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(5));
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(4));
+        /*
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(7));
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(6));
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(5));
+                GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(4));
+        */
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(3));
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(2));
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(1));
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(0));
 
+#if !STOP_PRIME
+#if !EPM_P9_TUNING
         // Make sure we are not forcing PFET for VDD or VCS off
         // vdd_pfet_force_state == 00 (Nop)
         // vcs_pfet_force_state == 00 (Nop)
@@ -441,6 +462,8 @@ p9_sgpe_stop_entry()
             return SGPE_STOP_ENTRY_VDD_PFET_NOT_IDLE;
         }
 
+#endif
+
         // Prepare PFET Controls
         // vdd_pfet_val/sel_override     = 0 (disbaled)
         // vcs_pfet_val/sel_override     = 0 (disbaled)
@@ -448,12 +471,11 @@ p9_sgpe_stop_entry()
         PK_TRACE("SE11.t");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, qloop), BITS64(4, 4) | BIT64(8));
 
-        // Power Off Core VDD
+        // Power Off Cache VDD/VDS
         // vdd_pfet_force_state = 01 (Force Voff)
         // vcs_pfet_force_state = 01 (Force Voff)
         PK_TRACE("SE11.u");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, qloop), BIT64(1) | BIT64(3));
-
         // Poll for power gate sequencer state: 0x8 (FSM Idle)
         PK_TRACE("SE11.v");
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index aa6e200a..6283fc53 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -65,7 +65,6 @@ p9_sgpe_stop_exit()
                                      STOP_ACT_DISABLE);
             MARK_TRAP(SX_LV11_WAKEUP_START)
 
-#if !STOP_PRIME
             PK_TRACE("Cache Poweron");
             p9_hcd_cache_poweron(qloop);
             MARK_TRAP(SX_POWERON_END)
@@ -74,14 +73,17 @@ p9_sgpe_stop_exit()
             p9_hcd_cache_chiplet_reset(qloop);
             MARK_TRAP(SX_CHIPLET_RESET_END)
 
+#if !STOP_PRIME
             PK_TRACE("Cache Gptr Time Initf");
             p9_hcd_cache_gptr_time_initf(qloop);
             MARK_TRAP(SX_GPTR_TIME_INITF_END)
+#endif
 
             PK_TRACE("Cache Dpll Setup");
             p9_hcd_cache_dpll_setup(qloop);
             MARK_TRAP(SX_DPLL_SETUP_END)
 
+#if !STOP_PRIME
             PK_TRACE("Cache Chiplet Init");
             p9_hcd_cache_chiplet_init(qloop);
             MARK_TRAP(SX_CHIPLET_INIT_END)
@@ -103,7 +105,6 @@ p9_sgpe_stop_exit()
             p9_hcd_cache_startclocks(qloop);
             MARK_TRAP(SX_STARTCLOCKS_END)
 
-            G_sgpe_stop_record.state[qloop].detail.q_act = 0;
         }
 
         if((G_sgpe_stop_record.state[qloop].detail.x0act >= STOP_LEVEL_8 &&
@@ -170,7 +171,9 @@ p9_sgpe_stop_exit()
             PK_TRACE("Cache OCC Runtime Scom");
             p9_hcd_cache_occ_runtime_scom(qloop);
             MARK_TRAP(SX_OCC_RUNTIME_SCOM_END)
+
 #endif
+            G_sgpe_stop_record.state[qloop].detail.q_act = 0;
         }
 
         for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
@@ -186,12 +189,16 @@ p9_sgpe_stop_exit()
                 GPE_GETSCOM(CME_SCOM_FLAGS, QUAD_ADDR_BASE|CME_ADDR_OFFSET_EX0,
                             ((qloop<<2)+cloop), scom_data);
             } while(!(scom_data & BIT64(0)));*/
+            // TODO PUT THE FOLLOWING TWO BEFORE CME_BOOT()
             // Change PPM Wakeup to CME
             GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CPMMR_CLR, ((qloop << 2) + cloop)),
                         BIT64(13));
             PK_TRACE("Doorbell1 the CME");
+            GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEMSG, ((qloop << 2) + cloop)),
+                        (BIT64(0)));
             GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB1_OR, ((qloop << 2) + cloop)),
                         BIT64(7));
+
         }
     }
 
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
index 0cdb575c..f6051236 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
@@ -194,8 +194,10 @@ p9_sgpe_stop_pig_type2_handler(void* arg, PkIrqId irq)
         pk_semaphore_post(&(G_sgpe_stop_record.sem[1]));
     }
 
-    if (!G_sgpe_stop_record.group.vector[0])
-    {
-        out32(OCB_OIMR1_CLR, BIT32(15));
-    }
+    /*
+        if (!G_sgpe_stop_record.group.vector[0])
+        {
+            out32(OCB_OIMR1_CLR, BIT32(15));
+        }
+    */
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
index 178185bb..b7a6194b 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
@@ -36,7 +36,7 @@
 
 #define STOP_PRIME          0
 #define SKIP_L3_PURGE       0
-#define SKIP_L3_PURGE_ABORT 0
+#define SKIP_L3_PURGE_ABORT 1
 
 // --------------------