CME/SGPE: Optimus Prime approves these upgrade of STOP images

Change-Id: Id1d699e416c2dcc34a2a5f442e009ce39e416d8c
Original-Change-Id: I2486797c6614213418624427fd0d225e2792d15c
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/23684
Tested-by: Jenkins Server
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

3 files changed, 309 insertions, 180 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index cede43da..ffeb59f1 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -51,8 +51,6 @@ extern CmeStopRecord G_cme_stop_record;
 int
 p9_cme_stop_entry()
 {
-
-    int             rc                  = 0;
     int             catchup_ongoing     = 0;
     int             entry_ongoing       = 1;
     uint8_t         target_level;
@@ -141,9 +139,11 @@ p9_cme_stop_entry()
 
         G_cme_stop_record.core_running &= ~core;
 
-        PK_TRACE("SE1: Request Stop Levels[%d %d]",
+        PK_TRACE("SE1: Stop Levels Request[%d %d] Actual[%d, %d]",
                  G_cme_stop_record.req_level_c0,
-                 G_cme_stop_record.req_level_c1);
+                 G_cme_stop_record.req_level_c1,
+                 G_cme_stop_record.act_level_c0,
+                 G_cme_stop_record.act_level_c1);
 
         // Return error if target STOP level == 1(Nap)
         if((core == CME_MASK_C0 &&
@@ -170,6 +170,11 @@ p9_cme_stop_entry()
         // Protect PPM Register Write
         CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(0));
 
+        PK_TRACE("SE2.g");
+        // Acknowledge the STOP Entry to PC with a pulse
+        out32(CME_LCL_SICR_OR,  core << SHIFT32(1));
+        out32(CME_LCL_SICR_CLR, core << SHIFT32(1));
+
         // set target_level from pm_state for both cores or just one core
         target_level = (core == CME_MASK_C0) ? G_cme_stop_record.req_level_c0 :
                        G_cme_stop_record.req_level_c1;
@@ -214,8 +219,8 @@ p9_cme_stop_entry()
                                     STOP_ACT_DISABLE);
         }
 
-        PK_TRACE("SE2: target_lv[%d], deeper_lv[%d], deeper_core[%d]",
-                 target_level, deeper_level, deeper_core);
+        PK_TRACE("SE2: core[%d], target_lv[%d], deeper_lv[%d], deeper_core[%d]",
+                 core, target_level, deeper_level, deeper_core);
 
         // Poll Infinitely for PCB Mux Grant
         // MF: change watchdog timer in pk to ensure forward progress
@@ -227,14 +232,11 @@ p9_cme_stop_entry()
         MARK_TRAP(SE_QUIESCE_CORE_INTF)
         //=============================
 
-        PK_TRACE("SE2.g");
-        // Acknowledge the STOP Entry to PC with a pulse
-        out32(CME_LCL_SICR_OR,  core << SHIFT32(1));
-        out32(CME_LCL_SICR_CLR, core << SHIFT32(1));
-
         /// Set LMCR bits 12/13, 14/15 (override disables)
-        out32(CME_LCL_LMCR_OR, ((core << SHIFT32(13)) | (core << SHIFT32(15))));
-
+#if SPWU_AUTO
+        out32(CME_LCL_LMCR_OR, (core << SHIFT32(13)));
+#endif
+        out32(CME_LCL_LMCR_OR, (core << SHIFT32(15)));
         PK_TRACE("SE2.h");
         // Raise Core-L2 + Core-CC Quiesces
         out32(CME_LCL_SICR_OR, (core << SHIFT32(7)) | (core << SHIFT32(9)));
@@ -265,16 +267,25 @@ p9_cme_stop_entry()
 
         PK_TRACE("SE2.l");
         // Stop Core Clocks
-        CME_PUTSCOM(C_CLK_REGION, core, 0x8FFC00000000E000);
+        CME_PUTSCOM(C_CLK_REGION, core, (CLK_STOP_CMD | CLK_REGION_ALL_BUT_PLL | CLK_THOLD_ALL));
 
         PK_TRACE("SE2.m");
 
         // Poll for core clocks stopped
         do
         {
-            CME_GETSCOM(C_CLOCK_STAT_SL, core, CME_SCOM_AND, scom_data);
+            CME_GETSCOM(C_CPLT_STAT0, core, CME_SCOM_AND, scom_data);
+        }
+        while((~scom_data) & BIT64(8));
+
+        PK_TRACE("Check core clock is stopped via CLOCK_STAT_SL[4-13]");
+        CME_GETSCOM(C_CLOCK_STAT_SL, core, CME_SCOM_AND, scom_data);
+
+        if (((~scom_data) & CLK_REGION_ALL_BUT_PLL) != 0)
+        {
+            PK_TRACE("Core clock stop failed");
+            pk_halt();
         }
-        while((scom_data & BITS64(4, 10)) != BITS64(4, 10));
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
@@ -305,9 +316,15 @@ p9_cme_stop_entry()
         // Assert skew sense to skewadjust Fence
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(22));
         // Assert Vital Fence
-        CME_PUTSCOM(C_CPLT_CTRL1_OR,  core, BIT64(3));
+        CME_PUTSCOM(C_CPLT_CTRL1_OR, core, BIT64(3));
         // Assert Regional Fences
-        CME_PUTSCOM(C_CPLT_CTRL1_OR, core, 0xFFFF700000000000);
+        CME_PUTSCOM(C_CPLT_CTRL1_OR, core, BITS64(4, 11));
+        /// @todo add DD1 attribute control
+        PK_TRACE("DD1 only: reset sdis_n(flushing LCBES condition workaround");
+        CME_PUTSCOM(C_CPLT_CONF0_CLEAR, core, BIT64(34));
+        /// @todo add VDM_ENABLE attribute control
+        PK_TRACE("Drop vdm enable via CPPM_VDMCR[0]");
+        CME_PUTSCOM(PPM_VDMCR_CLR, core, BIT64(0));
 
         PK_TRACE("SE2: Clock Sync Dropped");
 
@@ -365,43 +382,6 @@ p9_cme_stop_entry()
             entry_ongoing = 1;
         }
 
-        //===========================
-        MARK_TRAP(SE_IS0_BEGIN)
-        //===========================
-#if !SKIP_ABORT
-        out32(CME_LCL_EIMR_CLR, (core << SHIFT32(13)) |
-              (core << SHIFT32(15)) |
-              (core << SHIFT32(17)));
-        sync();
-        out32(CME_LCL_EIMR_OR,  (core << SHIFT32(13)) |
-              (core << SHIFT32(15)) |
-              (core << SHIFT32(17)));
-#endif
-        //===================
-        MARK_TRAP(SE_IS0_END)
-        //===================
-
-        core_aborted = core & G_cme_stop_record.core_running;
-
-        if (core_aborted && deeper_core)
-        {
-            if (core_aborted != deeper_core)
-            {
-                target_level = deeper_level;
-            }
-
-            deeper_core  = 0;
-        }
-
-        core = core & ~G_cme_stop_record.core_running;
-
-        if (!core)
-        {
-            core |= core_aborted;
-            entry_ongoing = 0;
-            break;
-        }
-
 #if !SKIP_ENTRY_CATCHUP
 
         if (catchup_ongoing)
@@ -462,6 +442,9 @@ p9_cme_stop_entry()
             //========================
         }
 
+        PK_TRACE("core[%d] running[%d] core_catchup[%d] origin_core[%d]",
+                 core, G_cme_stop_record.core_running, core_catchup, origin_core);
+
 #endif
 
     }
@@ -470,12 +453,51 @@ p9_cme_stop_entry()
     do
     {
 
-        // If we are done at STOP level 2 or aborted
+        // If we are done at STOP level 2
         if (!entry_ongoing)
         {
             break;
         }
 
+        //===========================
+        MARK_TRAP(SE_IS0_BEGIN)
+        //===========================
+#if !SKIP_ABORT
+        out32(CME_LCL_EIMR_CLR, (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+        sync();
+        out32(CME_LCL_EIMR_OR,  (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+#endif
+        //===================
+        MARK_TRAP(SE_IS0_END)
+        //===================
+
+        core_aborted = core &  G_cme_stop_record.core_running;
+        core         = core & ~G_cme_stop_record.core_running;
+
+        PK_TRACE("core[%d] running[%d] core_aborted[%d]",
+                 core, G_cme_stop_record.core_running, core_aborted);
+
+        if (!core)
+        {
+            core |= core_aborted;
+            entry_ongoing = 0;
+            break;
+        }
+
+        if (core_aborted && deeper_core)
+        {
+            if (core_aborted != deeper_core)
+            {
+                target_level = deeper_level;
+            }
+
+            deeper_core  = 0;
+        }
+
         PK_TRACE("SE2+:core[%d],deeper_core[%d],\
                   target_level[%d],deeper_level[%d]",
                  core, deeper_core, target_level, deeper_level);
@@ -483,75 +505,75 @@ p9_cme_stop_entry()
         //----------------------------------------------------------------------
         // STOP LEVEL 3
         //----------------------------------------------------------------------
+        /*
+                if (target_level == 3)
+                {
 
-        if (target_level == 3)
-        {
-
-            //==========================
-            MARK_TAG(SE_CORE_VMIN, core)
-            //==========================
-
-            PK_TRACE("SE3.a");
-            // Enable IVRM if not already
-
-            PK_TRACE("SE3.b");
-            // Drop to Vmin
-
-            if(core & CME_MASK_C0)
-            {
-                G_cme_stop_record.act_level_c0 = STOP_LEVEL_3;
-            }
+                    //==========================
+                    MARK_TAG(SE_CORE_VMIN, core)
+                    //==========================
 
-            if(core & CME_MASK_C1)
-            {
-                G_cme_stop_record.act_level_c1 = STOP_LEVEL_3;
-            }
+                    PK_TRACE("SE3.a");
+                    // Enable IVRM if not already
 
-            //===========================
-            MARK_TAG(SE_STOP3_DONE, core)
-            //===========================
+                    PK_TRACE("SE3.b");
+                    // Drop to Vmin
 
-            PK_TRACE("SE3.c");
-            // Update Stop History: In Core Stop Level 3
-            CME_STOP_UPDATE_HISTORY(core,
-                                    STOP_CORE_IS_GATED,
-                                    STOP_TRANS_COMPLETE,
-                                    target_level,
-                                    STOP_LEVEL_3,
-                                    STOP_REQ_DISABLE,
-                                    STOP_ACT_ENABLE);
+                    if(core & CME_MASK_C0)
+                    {
+                        G_cme_stop_record.act_level_c0 = STOP_LEVEL_3;
+                    }
 
-            // If both cores targeting different levels
-            // deeper core should have at least deeper stop level than 3
-            // only need to modify deeper core history if another one was done
-            if (deeper_core)
-            {
-                CME_STOP_UPDATE_HISTORY(deeper_core,
-                                        STOP_CORE_IS_GATED,
-                                        STOP_TRANS_ENTRY,
-                                        deeper_level,
-                                        STOP_LEVEL_2,
-                                        STOP_REQ_DISABLE,
-                                        STOP_ACT_ENABLE);
-                // from now on, proceed with only deeper core
-                core          = deeper_core;
-                target_level  = deeper_level;
-                deeper_level  = 0;
-                deeper_core   = 0;
-                entry_ongoing = 1;
-            }
-            else
-            {
-                entry_ongoing = 0;
-            }
+                    if(core & CME_MASK_C1)
+                    {
+                        G_cme_stop_record.act_level_c1 = STOP_LEVEL_3;
+                    }
 
-            // If we are done at STOP level 3
-            if (!entry_ongoing)
-            {
-                break;
-            }
-        }
+                    //===========================
+                    MARK_TAG(SE_STOP3_DONE, core)
+                    //===========================
+
+                    PK_TRACE("SE3.c");
+                    // Update Stop History: In Core Stop Level 3
+                    CME_STOP_UPDATE_HISTORY(core,
+                                            STOP_CORE_IS_GATED,
+                                            STOP_TRANS_COMPLETE,
+                                            target_level,
+                                            STOP_LEVEL_3,
+                                            STOP_REQ_DISABLE,
+                                            STOP_ACT_ENABLE);
+
+                    // If both cores targeting different levels
+                    // deeper core should have at least deeper stop level than 3
+                    // only need to modify deeper core history if another one was done
+                    if (deeper_core)
+                    {
+                        CME_STOP_UPDATE_HISTORY(deeper_core,
+                                                STOP_CORE_IS_GATED,
+                                                STOP_TRANS_ENTRY,
+                                                deeper_level,
+                                                STOP_LEVEL_2,
+                                                STOP_REQ_DISABLE,
+                                                STOP_ACT_ENABLE);
+                        // from now on, proceed with only deeper core
+                        core          = deeper_core;
+                        target_level  = deeper_level;
+                        deeper_level  = 0;
+                        deeper_core   = 0;
+                        entry_ongoing = 1;
+                    }
+                    else
+                    {
+                        entry_ongoing = 0;
+                    }
 
+                    // If we are done at STOP level 3
+                    if (!entry_ongoing)
+                    {
+                        break;
+                    }
+                }
+        */
         //----------------------------------------------------------------------
         // STOP LEVEL 4
         //----------------------------------------------------------------------
@@ -626,7 +648,7 @@ p9_cme_stop_entry()
                                 STOP_ACT_ENABLE);
 
         // If both cores targeting different levels
-        // deeper core should have at least deeper stop level than 2
+        // deeper core should have at least deeper stop level than 4
         // only need to modify deeper core history if another one was done
         if (deeper_core && !entry_ongoing)
         {
@@ -645,6 +667,12 @@ p9_cme_stop_entry()
             entry_ongoing = 1;
         }
 
+        // If we are done at STOP level 4
+        if (!entry_ongoing)
+        {
+            break;
+        }
+
         //===========================
         MARK_TRAP(SE_IS1_BEGIN)
         //===========================
@@ -661,19 +689,11 @@ p9_cme_stop_entry()
         MARK_TRAP(SE_IS1_END)
         //===================
 
-        core_aborted = core & G_cme_stop_record.core_running;
-
-        if (core_aborted && deeper_core)
-        {
-            if (core_aborted != deeper_core)
-            {
-                target_level = deeper_level;
-            }
-
-            deeper_core  = 0;
-        }
+        core_aborted = core &  G_cme_stop_record.core_running;
+        core         = core & ~G_cme_stop_record.core_running;
 
-        core = core & ~G_cme_stop_record.core_running;
+        PK_TRACE("core[%d] running[%d] core_aborted[%d]",
+                 core, G_cme_stop_record.core_running, core_aborted);
 
         if (!core)
         {
@@ -682,10 +702,14 @@ p9_cme_stop_entry()
             break;
         }
 
-        // If we are done at STOP level 4 or aborted
-        if (!entry_ongoing)
+        if (core_aborted && deeper_core)
         {
-            break;
+            if (core_aborted != deeper_core)
+            {
+                target_level = deeper_level;
+            }
+
+            deeper_core  = 0;
         }
 
         PK_TRACE("SE4+:core[%d],deeper_core[%d],\
@@ -696,22 +720,21 @@ p9_cme_stop_entry()
         // STOP LEVEL 5 (preparsion of STOP LEVEL 8 and above)
         //----------------------------------------------------------------------
 
-        // block all wake up before purge L2,
-        // this is last chance either core can exit
-        //out32(CME_LCL_EIMR_OR, BITS32(12, 6));
-
         if ((G_cme_stop_record.req_level_c0 >= STOP_LEVEL_8) &&
             (G_cme_stop_record.req_level_c1 >= STOP_LEVEL_8))
         {
 
-            //=========================
-            MARK_TAG(SE_PURGE_L2, core)
-            //=========================
+            //================================
+            MARK_TAG(SE_PURGE_L2, CME_MASK_BC)
+            //================================
 
-            // Assert L2+NCU Purges(chtm purge will be done in SGPE),
-            // and NCU tlbie quiesce
+            // Assert L2+NCU Purge and NCU tlbie quiesce
+            // (chtm purge will be done in SGPE),
+            // insert tlbie quiesce before ncu purge to avoid window condition
+            // of ncu traffic still happening when purging starts
             PK_TRACE("SE5.1a");
-            out32(CME_LCL_SICR_OR, BIT32(18) | BIT32(21) | BIT32(22));
+            out32(CME_LCL_SICR_OR, BIT32(18) | BIT32(21));
+            out32(CME_LCL_SICR_OR, BIT32(22));
 
             // todo: poll for tlbie quiesce done?
 
@@ -722,12 +745,23 @@ p9_cme_stop_entry()
             {
 #if !SKIP_L2_PURGE_ABORT
 
-                if(in32(CME_LCL_EINR) & BITS32(12, 6))
+                if (!core_aborted &&
+                    (in32(CME_LCL_EINR) & BITS32(12, 6)))
                 {
+                    if (in32(CME_LCL_EINR) & IRQ_VEC_WAKE_C0)
+                    {
+                        core_aborted |= CME_MASK_C0;
+                    }
 
-                    //===============================
-                    MARK_TAG(SE_PURGE_L2_ABORT, core)
-                    //===============================
+                    if (in32(CME_LCL_EINR) & IRQ_VEC_WAKE_C1)
+                    {
+                        core_aborted |= CME_MASK_C1;
+                    }
+
+                    PK_TRACE("L2 Purge aborted by core[%d]", core_aborted);
+                    //=======================================
+                    MARK_TAG(SE_PURGE_L2_ABORT, core_aborted)
+                    //=======================================
                     // abort L2+NCU purges
                     out32(CME_LCL_SICR_OR, BIT32(19) | BIT32(23));
                 }
@@ -736,15 +770,24 @@ p9_cme_stop_entry()
             }
             while((in32(CME_LCL_EISR) & BITS32(22, 2)) != BITS32(22, 2));
 
-            // Deassert L2+NCU Purges, their possible aborts, NCU tlbie quiesce
+            // Deassert L2+NCU Purges, their possible aborts
             PK_TRACE("SE5.1c");
-            out32(CME_LCL_SICR_CLR, BITS32(18, 6));
+            out32(CME_LCL_SICR_CLR, (BITS32(18, 2) | BITS32(22, 2)));
 
             PK_TRACE("SE5.1: L2/NCU/CHTM Purged");
 
-            //==============================
-            MARK_TAG(SE_PURGE_L2_DONE, core)
-            //==============================
+            //===============================================================
+            MARK_TAG(SE_PURGE_L2_DONE, core_aborted ? core_aborted : CME_MASK_BC)
+            //===============================================================
+
+            if (core !=  core_aborted)
+            {
+                core &= ~core_aborted;
+            }
+            else
+            {
+                break;
+            }
         }
 
         //=============================
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 61844b9c..322b829c 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -49,15 +49,19 @@
 
 #include "p9_stop_common.h"
 
+#define EQ_RING_FENCE_MASK_LATCH 0x10010008
 #define EQ_SYNC_CONFIG           0x10030000
 #define EQ_OPCG_ALIGN            0x10030001
 #define EQ_SCAN_REGION_TYPE      0x10030005
 #define EQ_CLK_REGION            0x10030006
 #define EQ_CLOCK_STAT_SL         0x10030008
+#define EQ_THERM_MODE_REG        0x1005000F
 
 #define EQ_BIST                  0x100F000B
+#define EQ_HANG_PULSE_6_REG      0x100F0026
 #define EQ_NET_CTRL0_WAND        0x100F0041
 #define EQ_NET_CTRL0_WOR         0x100F0042
+#define C_NET_CTRL0              0x200F0040
 #define C_NET_CTRL0_WOR          0x200F0042
 #define EQ_NET_CTRL1_WAND        0x100F0045
 #define EQ_NET_CTRL1_WOR         0x100F0046
@@ -66,6 +70,8 @@
 #define EQ_CPLT_CTRL0_CLEAR      0x10000020
 #define EQ_CPLT_CTRL1_OR         0x10000011
 #define EQ_CPLT_CTRL1_CLEAR      0x10000021
+#define EQ_CPLT_CONF0_OR         0x10000018
+#define EQ_CPLT_CONF0_CLEAR      0x10000028
 #define EQ_CPLT_STAT0            0x10000100
 
 #define EQ_QPPM_DPLL_CTRL_CLEAR  0x100F0153
@@ -88,8 +94,16 @@
 #define EX_PM_LCO_DIS_REG        0x10011816
 #define EX_PM_L2_RCMD_DIS_REG    0x10011818
 
-#define SGPE_STOP_L2_CLOCK_REGION(ex) (ex << SHIFT64(9))
-#define SGPE_STOP_L3_CLOCK_REGION(ex) (ex << SHIFT64(7))
+#define PERV_CPLT_CTRL0_OR       0x10000010
+#define PERV_CPLT_CTRL0_CLEAR    0x10000020
+#define PERV_CPLT_CTRL1_OR       0x10000011
+#define PERV_CPLT_CTRL1_CLEAR    0x10000021
+#define PERV_OPCG_REG0           0x10030002
+#define PERV_OPCG_REG1           0x10030003
+#define PERV_SCAN_REGION_TYPE    0x10030005
+#define PERV_CLK_REGION          0x10030006
+#define PERV_BIST                0x1003000B
+#define PERV_CPLT_STAT0          0x10000100
 
 /// Macro to update STOP History
 #define SGPE_STOP_UPDATE_HISTORY(id,base,gated,trans,req_l,act_l,req_e,act_e) \
@@ -127,6 +141,16 @@ enum SGPE_STOP_EVENT_LEVELS
     LEVEL_EQ_BASE                     = 11
 };
 
+enum SGPE_STOP_PSCOM_MASK
+{
+    PSCOM_MASK_ALL_L2                 = BITS64(2, 2) | BITS64(10, 2),
+    PSCOM_MASK_EX0_L2                 = BIT64(2) | BIT64(10),
+    PSCOM_MASK_EX1_L2                 = BIT64(3) | BIT64(11),
+    PSCOM_MASK_EX0_L3                 = BIT64(4) | BIT64(6) | BIT64(8),
+    PSCOM_MASK_EX1_L3                 = BIT64(5) | BIT64(7) | BIT64(9)
+};
+
+
 enum SGPE_STOP_VECTOR_INDEX
 {
     VECTOR_EXIT                       = 0,
@@ -144,6 +168,8 @@ typedef struct
     uint8_t act_state_x0;
     uint8_t act_state_x1;
     uint8_t act_state_q;
+    // both cme_flags: first(0:3) | enable(4:7)
+    uint8_t cme_flags;
 } sgpe_state_t;
 
 typedef struct
@@ -176,16 +202,17 @@ void p9_sgpe_stop_exit_thread(void*);
 int  p9_sgpe_stop_entry();
 int  p9_sgpe_stop_exit();
 
+int  p9_hcd_cache_scan0(uint32_t, uint64_t, uint64_t);
 int  p9_hcd_cache_poweron(uint32_t);
-int  p9_hcd_cache_chiplet_reset(uint32_t);
+int  p9_hcd_cache_chiplet_reset(uint32_t, uint32_t);
 int  p9_hcd_cache_gptr_time_initf(uint32_t);
 int  p9_hcd_cache_dpll_setup(uint32_t);
 int  p9_hcd_cache_chiplet_init(uint32_t);
 int  p9_hcd_cache_repair_initf(uint32_t);
-int  p9_hcd_cache_arrayinit(uint32_t);
+int  p9_hcd_cache_arrayinit(uint32_t, uint32_t ex);
 int  p9_hcd_cache_initf(uint32_t);
 int  p9_hcd_cache_startclocks(uint32_t, uint32_t);
-int  p9_hcd_cache_l2_startclocks(uint32_t, uint32_t);
+int  p9_hcd_cache_l2_startclocks(uint32_t, uint32_t, uint32_t);
 int  p9_hcd_cache_scominit(uint32_t);
 int  p9_hcd_cache_scomcust(uint32_t);
 int  p9_hcd_cache_ras_runtime_scom(uint32_t);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index 5876f51f..bd771b4e 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -33,7 +33,6 @@ p9_sgpe_stop_entry()
 {
     int          entry_ongoing[2] = {0, 0};
     int          l3_purge_aborted = 0;
-    int          rc = 0;
     uint32_t     ex = 0;
     uint32_t     qloop;
     uint32_t     cloop;
@@ -42,6 +41,7 @@ p9_sgpe_stop_entry()
     uint32_t     qentry;
     uint32_t     loop;
     uint64_t     scom_data;
+    uint64_t     temp_data;
     ppm_sshsrc_t hist;
 
     //================================
@@ -94,8 +94,7 @@ p9_sgpe_stop_entry()
                 BIT32((qloop << 1) + 1);
         }
 
-        if (G_sgpe_stop_record.state[qloop].act_state_q <
-            G_sgpe_stop_record.state[qloop].req_state_q &&
+        if (G_sgpe_stop_record.state[qloop].act_state_q <  LEVEL_EQ_BASE &&
             G_sgpe_stop_record.state[qloop].req_state_q >= LEVEL_EQ_BASE)
         {
             G_sgpe_stop_record.group.quad[VECTOR_ENTRY] |= BIT32(qloop);
@@ -115,10 +114,10 @@ p9_sgpe_stop_entry()
                      G_sgpe_stop_record.state[qloop].act_state_x0,
                      G_sgpe_stop_record.state[qloop].act_state_x1);
 
-            PK_TRACE("req: x0lv[%d]x1lv[%d]qlv[%d]",
+            PK_TRACE("req: qlv[%d]x0lv[%d]x1lv[%d]",
+                     G_sgpe_stop_record.state[qloop].req_state_q,
                      G_sgpe_stop_record.state[qloop].req_state_x0,
-                     G_sgpe_stop_record.state[qloop].req_state_x1,
-                     G_sgpe_stop_record.state[qloop].req_state_q);
+                     G_sgpe_stop_record.state[qloop].req_state_x1);
         }
     }
 
@@ -202,6 +201,31 @@ p9_sgpe_stop_entry()
 
         PPE_WAIT_CORE_CYCLES(loop, 256)
 
+        PK_TRACE("set partial bad l2/l3 and stopping/stoped l2 pscom masks");
+        scom_data = 0;
+
+        if (!(G_sgpe_stop_record.group.ex_l[VECTOR_CONFIG] & BIT32(qloop)))
+        {
+            scom_data |= (PSCOM_MASK_EX0_L2 | PSCOM_MASK_EX0_L3);
+        }
+        else if ((ex & FST_EX_IN_QUAD) ||
+                 (G_sgpe_stop_record.state[qloop].act_state_x0 >= LEVEL_EX_BASE))
+        {
+            scom_data |= PSCOM_MASK_EX0_L2;
+        }
+
+        if (!(G_sgpe_stop_record.group.ex_r[VECTOR_CONFIG] & BIT32(qloop)))
+        {
+            scom_data |= (PSCOM_MASK_EX1_L2 | PSCOM_MASK_EX1_L3);
+        }
+        else if ((ex & SND_EX_IN_QUAD) ||
+                 (G_sgpe_stop_record.state[qloop].act_state_x1 >= LEVEL_EX_BASE))
+        {
+            scom_data |= PSCOM_MASK_EX1_L2;
+        }
+
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_RING_FENCE_MASK_LATCH, qloop), scom_data);
+
         PK_TRACE("SE8.b");
         // Set all bits to zero prior stop cache clocks
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
@@ -209,16 +233,25 @@ p9_sgpe_stop_entry()
         PK_TRACE("SE8.c");
         // Stop L2 Clocks
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop),
-                    (0x800000000000E000 | ((uint64_t)ex << SHIFT64(9))));
+                    (CLK_STOP_CMD | CLK_THOLD_ALL |
+                     ((uint64_t)ex << SHIFT64(9))));
 
         PK_TRACE("SE8.d");
-
         // Poll for L2 clocks stopped
+
         do
         {
-            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_STAT0, qloop), scom_data);
+        }
+        while((~scom_data) & BIT64(8));
+
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
+
+        if (((~scom_data) & ((uint64_t)ex << SHIFT64(9))) != 0)
+        {
+            PK_TRACE("L2 clock stop failed");
+            pk_halt();
         }
-        while(((scom_data >> SHIFT64(9)) & ex) != ex);
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
@@ -460,16 +493,22 @@ p9_sgpe_stop_entry()
             }
 
 #endif
+            scom_data = 0;
+            temp_data = 0;
 
             if (ex & FST_EX_IN_QUAD)
+            {
                 GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, 0),
                             scom_data);
+            }
 
             if (ex & SND_EX_IN_QUAD)
+            {
                 GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, 1),
-                            scom_data);
+                            temp_data);
+            }
         }
-        while(scom_data & BIT64(0));
+        while((scom_data | temp_data) & BIT64(0));
 
         if (l3_purge_aborted)
         {
@@ -565,18 +604,29 @@ p9_sgpe_stop_entry()
         PK_TRACE("SE11.m");
         // Stop Cache Clocks
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop),
-                    0x8C3E00000000E000 | ((uint64_t)ex << SHIFT64(7)));
+                    (CLK_STOP_CMD | CLK_THOLD_ALL |
+                     CLK_REGION_ALL_BUT_EX        |
+                     ((uint64_t)ex << SHIFT64(7)) |
+                     ((uint64_t)ex << SHIFT64(13))));
 
         PK_TRACE("SE11.n");
-
         // Poll for Cache clocks stopped
+
         do
         {
-            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
+            GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_STAT0, qloop), scom_data);
+        }
+        while((~scom_data) & BIT64(8));
+
+        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
+
+        if (((~scom_data) & (CLK_REGION_ALL_BUT_EX        |
+                             ((uint64_t)ex << SHIFT64(7)) |
+                             ((uint64_t)ex << SHIFT64(13)))) != 0)
+        {
+            PK_TRACE("Cache clock stop failed");
+            pk_halt();
         }
-        while((scom_data &
-               (BITS64(4, 2) | ((uint64_t)ex << SHIFT64(7)) | BITS64(10, 5))) !=
-              (BITS64(4, 2) | ((uint64_t)ex << SHIFT64(7)) | BITS64(10, 5)));
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
@@ -589,8 +639,16 @@ p9_sgpe_stop_entry()
         // Assert Vital Fence
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop), BIT64(3));
         // Raise Partial Good Fences
+        // Must cover partial bad fences as well or powerbus error will raise
+        // Note: Stop11 will lose all the fences so here needs to assert them
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop),
-                    0xFFFF700000000000);
+                    (CLK_REGION_ALL_BUT_EX        |
+                     ((uint64_t)ex << SHIFT64(7)) |
+                     ((uint64_t)ex << SHIFT64(9)) |
+                     ((uint64_t)ex << SHIFT64(13))));
+        /// @todo add VDM_ENABLE attribute control
+        PK_TRACE("Assert vdm enable via CPPM_VDMCR[0]");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_VDMCR_CLR, qloop), BIT64(0));
 
         //=========================================
         MARK_TAG(SE_POWER_OFF_CACHE, (32 >> qloop))
@@ -603,7 +661,10 @@ p9_sgpe_stop_entry()
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(16));
 
         // L3 edram shutdown
-        // todo: not EPM flag for delay
+        // QCCR[0/4] EDRAM_ENABLE_DC
+        // QCCR[1/5] EDRAM_VWL_ENABLE_DC
+        // QCCR[2/6] L3_EX0/1_EDRAM_VROW_VBLH_ENABLE_DC
+        // QCCR[3/7] EDRAM_VPP_ENABLE_DC
         PK_TRACE("SE11.r");
 
         if (ex & SND_EX_IN_QUAD)
@@ -722,8 +783,6 @@ p9_sgpe_stop_entry()
         }
     }
 
-    // Enable Type2/3/6 Interrupt
-    out32(OCB_OIMR1_CLR, (BITS32(15, 2) | BIT32(19)));
     //============================
     MARK_TRAP(ENDSCOPE_STOP_ENTRY)
     //============================