CME/SGPE: yet another updating commit for stop images

patchset 1: fix EISTR in stop interrupt handler
patchset 2: add QPMMR[21-26] setup in sgpe(for John)
patchset 3: add DD1 VCS workaround to sgpe
patchset 4: add common header file change for patch3
patchset 5: fix an error in vcs workaround
patchset 6: reorganize the compiler flags
patchset 7: fix a typo in 6, and rebase to fix jenkins
patchset 8: reorganize the compiler flags(cme)
            remove ppm write protection of cme
patchset 9: fix a typo in sgpe_exit qssr reporting
patchset 10:add extra dd1 vcs workaround
patchset 11:rebase
patchset 12:fix cme interrupt handler(marker changed)
patchset 13:fix compiler error in 12
            turn on sgpe kernel trace
patchset 14:fix again
patchset 15:fix wake/stop priority group
patchset 16:fix markers for EPM
patchset 17:reformat CME PK_TRACEs
patchset 18:reformat SGPE PK_TRACEs
patchset 19:fix some typo in 18
patchset 20:fix VDM scom fail via INTERPPM settings
patchset 21:core hist in sgpe based on partial good
            fix vcs workaround in stop image
            add qloop limiter in sgpe exit
            add stop level mapping support
patchset 22:enable vcs workaround by default
patchset 23:add dpll_initf
            fix dpll_setup bit(9)
            make epm skip workaround doesnt need
patchset 24:fix stop2 exit express
            fix some old compiler flags
patchset 25:add dd1 doorbell workaround
            add comment on serialize quad PFETs
patchset 26:fix cme_boot() on repeat booting one CME
patchset 27:increase sgpe_exit_thread stack size
patchset 28:sdis_n set/reset becomes permanent
patchset 29:add skewadjust procedures
patchset 30:add queue scom mode init
patchset 31:using ppe_scom instead of fapi_scom
patchset 32:continue protect cme on partial bad core
patchset 33:add support for EX_L3_MODE_REG1 setup
patchset 34:using real time qcsr for l3-lco victem
patchset 35:fix a bug in patchset 32
patchset 36:Increased the CME/SGPE Hcode defines.

Change-Id: Ib2a9ab543e018fbd5832a3342f3ab39cf3da436d
Original-Change-Id: Id3fd4d4e0d9740a7903c913fa8fc80b6cee55ff9
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/30925
Dev-Ready: YUE DU <daviddu@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: AMIT KUMAR <akumar3@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

3 files changed, 353 insertions, 248 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index 29fefc38..2c3ea431 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -70,7 +70,7 @@ p9_cme_stop_entry()
     ppm_pig_t       pig;
 
     //--------------------------------------------------------------------------
-    // BEGIN OF STOP ENTRY
+    PK_TRACE_INF("+++++ +++++ BEGIN OF STOP ENTRY +++++ +++++");
     //--------------------------------------------------------------------------
 
     // First we need to determine which of the two STOP interrupts fired.
@@ -84,14 +84,13 @@ p9_cme_stop_entry()
     core = core & G_cme_stop_record.core_enabled &
            G_cme_stop_record.core_running;
 
-    PK_TRACE("SE0: Core Select[%d] Enabled[%d] Running[%d]",
-             core, G_cme_stop_record.core_enabled,
-             G_cme_stop_record.core_running);
+    PK_TRACE_INF("Check: Core Select[%d] Enabled[%d] Running[%d]",
+                 core, G_cme_stop_record.core_enabled,
+                 G_cme_stop_record.core_running);
 
-    // Return error if None of both fired
     if (!core)
     {
-        PK_TRACE("Error: no pm_active fired");
+        PK_TRACE_INF("ERROR: No PM_ACTIVE Fired From a Running and Enabled Core. HALT CME!");
         pk_halt();
     }
 
@@ -102,10 +101,6 @@ p9_cme_stop_entry()
     do   // while(0) loop for stop flow control
     {
 
-        //----------------------------------------------------------------------
-        // STOP LEVEL 1 (should be done by hardware)
-        //----------------------------------------------------------------------
-
         // Read SISR for pm_state_cX
         pm_states = in32_sh(CME_LCL_SISR);
 
@@ -136,6 +131,31 @@ p9_cme_stop_entry()
                 G_cme_stop_record.act_level_c0 = STOP_LEVEL_1;
                 core_stop1 |= CME_MASK_C0;
             }
+
+            if ((G_cme_stop_record.header_flags & MAP_11_TO_8) &&
+                (G_cme_stop_record.req_level_c0 == STOP_LEVEL_11))
+            {
+
+                G_cme_stop_record.req_level_c0 = STOP_LEVEL_8;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_8_TO_5) &&
+                (G_cme_stop_record.req_level_c0 == STOP_LEVEL_8))
+            {
+                G_cme_stop_record.req_level_c0 = STOP_LEVEL_5;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_5_TO_4) &&
+                (G_cme_stop_record.req_level_c0 == STOP_LEVEL_5))
+            {
+                G_cme_stop_record.req_level_c0 = STOP_LEVEL_4;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_4_TO_2) &&
+                (G_cme_stop_record.req_level_c0 == STOP_LEVEL_4))
+            {
+                G_cme_stop_record.req_level_c0 = STOP_LEVEL_2;
+            }
         }
 
         if (core & CME_MASK_C1)
@@ -148,24 +168,56 @@ p9_cme_stop_entry()
                 G_cme_stop_record.act_level_c1 = STOP_LEVEL_1;
                 core_stop1 |= CME_MASK_C1;
             }
+
+            if ((G_cme_stop_record.header_flags & MAP_11_TO_8) &&
+                (G_cme_stop_record.req_level_c1 == STOP_LEVEL_11))
+            {
+                G_cme_stop_record.req_level_c1 = STOP_LEVEL_8;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_8_TO_5) &&
+                (G_cme_stop_record.req_level_c1 == STOP_LEVEL_8))
+            {
+                G_cme_stop_record.req_level_c1 = STOP_LEVEL_5;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_5_TO_4) &&
+                (G_cme_stop_record.req_level_c1 == STOP_LEVEL_5))
+            {
+                G_cme_stop_record.req_level_c1 = STOP_LEVEL_4;
+            }
+
+            if ((G_cme_stop_record.header_flags & MAP_4_TO_2) &&
+                (G_cme_stop_record.req_level_c1 == STOP_LEVEL_4))
+            {
+                G_cme_stop_record.req_level_c1 = STOP_LEVEL_2;
+            }
         }
 
-        G_cme_stop_record.core_running &= ~core;
+        PK_TRACE_INF("Check: Stop Levels Request[%d %d] Actual[%d, %d]",
+                     G_cme_stop_record.req_level_c0,
+                     G_cme_stop_record.req_level_c1,
+                     G_cme_stop_record.act_level_c0,
+                     G_cme_stop_record.act_level_c1);
 
-        PK_TRACE("SE1: Stop Levels Request[%d %d] Actual[%d, %d]",
-                 G_cme_stop_record.req_level_c0,
-                 G_cme_stop_record.req_level_c1,
-                 G_cme_stop_record.act_level_c0,
-                 G_cme_stop_record.act_level_c1);
+        // Mark core as to be stopped
+        G_cme_stop_record.core_running &= ~core;
 
-        // Return error if target STOP level == 1(Nap)
         if(core_stop1)
         {
-#if HW386841_PLS_SRR1_DLS_STOP1_FIX
-            // Acknowledge the STOP Entry to PC with a pulse
+            PK_TRACE_INF("Check: core[%d] core_stop1[%d]", core, core_stop1);
+
+#if HW386841_DD1_PLS_SRR1_DLS_STOP1_FIX
+
+            //----------------------------------------------------------------------
+            PK_TRACE_INF("+++++ +++++ STOP LEVEL 1 ENTRY +++++ +++++");
+            //----------------------------------------------------------------------
+
+            PK_TRACE("Pulse STOP entry acknowledgement to PC via SICR[0/1]");
             out32(CME_LCL_SICR_OR,  core_stop1 << SHIFT32(1));
             out32(CME_LCL_SICR_CLR, core_stop1 << SHIFT32(1));
 
+            PK_TRACE("Update STOP history: in core stop level 1");
             CME_STOP_UPDATE_HISTORY(core_stop1,
                                     STOP_CORE_IS_GATED,
                                     STOP_TRANS_COMPLETE,
@@ -182,25 +234,24 @@ p9_cme_stop_entry()
             }
 
 #else
-            PK_TRACE("Error: stop 1 requested to hcode");
+
+            // Nap should be done by hardware when auto_stop1 is enabled
+            // Halt on error if target STOP level == 1(Nap)
+            PK_TRACE_INF("ERROR: Stop 1 Requested to CME When AUTO_STOP1 Enabled, HALT CME!");
             pk_halt();
+
 #endif
+
         }
 
         //----------------------------------------------------------------------
-        // STOP LEVEL 2
+        PK_TRACE_INF("+++++ +++++ STOP LEVEL 2 ENTRY +++++ +++++");
         //----------------------------------------------------------------------
 
-        PK_TRACE("SE2.c");
-        // Request PCB Mux
+        PK_TRACE("Request PCB mux via SICR[10/11]");
         out32(CME_LCL_SICR_OR, core << SHIFT32(11));
 
-        PK_TRACE("SE2.d");
-        // Protect PPM Register Write
-        CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(0));
-
-        PK_TRACE("SE2.g");
-        // Acknowledge the STOP Entry to PC with a pulse
+        PK_TRACE("Pulse STOP entry acknowledgement to PC via SICR[0/1]");
         out32(CME_LCL_SICR_OR,  core << SHIFT32(1));
         out32(CME_LCL_SICR_CLR, core << SHIFT32(1));
 
@@ -225,8 +276,7 @@ p9_cme_stop_entry()
             }
         }
 
-        PK_TRACE("SE2.e");
-        // Update STOP History: In Transition of Entry
+        PK_TRACE("Update STOP history: in transition of entry");
         // Set req_level_level to target_level of either both or just one core
         CME_STOP_UPDATE_HISTORY(core,
                                 STOP_CORE_READY_RUN,
@@ -248,30 +298,31 @@ p9_cme_stop_entry()
                                     STOP_ACT_DISABLE);
         }
 
-        PK_TRACE("SE2: core[%d], target_lv[%d], deeper_lv[%d], deeper_core[%d]",
-                 core, target_level, deeper_level, deeper_core);
+        PK_TRACE_INF("Check: core[%d] target_lv[%d] deeper_lv[%d] deeper_core[%d]",
+                     core, target_level, deeper_level, deeper_core);
 
         // Poll Infinitely for PCB Mux Grant
         // MF: change watchdog timer in pk to ensure forward progress
         while((core & (in32(CME_LCL_SISR) >> SHIFT32(11))) != core);
 
-        PK_TRACE("SE2: PCB Mux Granted");
+        PK_TRACE_INF("SE2.A: PCB Mux Granted");
 
         //=============================
         MARK_TRAP(SE_QUIESCE_CORE_INTF)
         //=============================
 
-        /// Set LMCR bits 12/13, 14/15 (override disables)
+        PK_TRACE("Assert halt STOP override disable via LMCR[14/15]");
         out32(CME_LCL_LMCR_OR, (core << SHIFT32(15)));
+
 #if SPWU_AUTO
+        PK_TRACE("Assert auto special wakeup disable via LMCR[12/13]");
         out32(CME_LCL_LMCR_OR, (core << SHIFT32(13)));
 #endif
 
-        PK_TRACE("SE2.h");
-        // Raise Core-L2 + Core-CC Quiesces
+        PK_TRACE("Assert core-L2 + core-CC quiesces via SICR[6/7,8/9]");
         out32(CME_LCL_SICR_OR, (core << SHIFT32(7)) | (core << SHIFT32(9)));
 
-        PK_TRACE("SE2.i");
+        PK_TRACE("Poll for L2 interface quiesced via SISR[30/31]");
 
         do
         {
@@ -283,25 +334,24 @@ p9_cme_stop_entry()
         // MF: verify generate FCB otherwise math is wrong.
         PPE_WAIT_CORE_CYCLES(loop, 512)
 
+        PK_TRACE_INF("SE2.B: Interfaces Quiesced");
+
         //==========================
         MARK_TRAP(SE_STOP_CORE_CLKS)
         //==========================
 
-        PK_TRACE("SE2.j");
-        // Raise Core Chiplet Fence
+        PK_TRACE("Assert core chiplet fence via NET_CTRL0[18]");
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(18));
 
-        PK_TRACE("SE2.k");
-        // Set all bits to zero prior stop core clocks
+        PK_TRACE("Clear SCAN_REGION_TYPE prior to stop core clocks");
         CME_PUTSCOM(C_SCAN_REGION_TYPE, core, 0);
 
-        PK_TRACE("SE2.l");
-        // Stop Core Clocks
-        CME_PUTSCOM(C_CLK_REGION, core, (CLK_STOP_CMD | CLK_REGION_ALL_BUT_PLL | CLK_THOLD_ALL));
+        PK_TRACE("Stop Core Clocks via CLK_REGION");
+        CME_PUTSCOM(C_CLK_REGION, core,
+                    (CLK_STOP_CMD | CLK_REGION_ALL_BUT_PLL | CLK_THOLD_ALL));
 
-        PK_TRACE("SE2.m");
+        PK_TRACE("Poll for core clocks stopped via CPLT_STAT0[8]");
 
-        // Poll for core clocks stopped
         do
         {
             CME_GETSCOM(C_CPLT_STAT0, core, CME_SCOM_AND, scom_data);
@@ -313,52 +363,51 @@ p9_cme_stop_entry()
 
         if (((~scom_data) & CLK_REGION_ALL_BUT_PLL) != 0)
         {
-            PK_TRACE("Core clock stop failed");
+            PK_TRACE_INF("ERROR: Core Clock Stop Failed. HALT CME!");
             pk_halt();
         }
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
 
-        PK_TRACE("SE2: Core Clock Stopped");
+        PK_TRACE_INF("SE2.C: Core Clock Stopped");
 
         //==============================
         MARK_TRAP(SE_STOP_CORE_GRID)
         //==============================
 
-        PK_TRACE("SE2.n");
-        // Drop clock sync enable before switch to refclk
+        PK_TRACE("Drop clock sync enable before switch to refclk via CACCR[15]");
         CME_PUTSCOM(CPPM_CACCR_CLR, core, BIT64(15));
 
-        PK_TRACE("SE2.o");
+        PK_TRACE("Poll for clock sync done to drop via CACSR[13]");
 
-        // Poll for clock sync done to drop
         do
         {
             CME_GETSCOM(CPPM_CACSR, core, CME_SCOM_OR, scom_data);
         }
         while(scom_data & BIT64(13));
 
-        PK_TRACE("SE2.p");
-        // Switch glsmux to refclk to save clock grid power
+        PK_TRACE("Switch glsmux to refclk to save clock grid power via CGCR[3]");
         CME_PUTSCOM(C_PPM_CGCR, core, 0);
 
-        // Assert skew sense to skewadjust Fence
+        PK_TRACE("Assert skew sense to skewadjust fence via NET_CTRL0[22]");
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(22));
-        // Assert Vital Fence
+
+        PK_TRACE("Assert vital fence via CPLT_CTRL1[3]");
         CME_PUTSCOM(C_CPLT_CTRL1_OR, core, BIT64(3));
-        // Assert Regional Fences
+
+        PK_TRACE("Assert regional fences via CPLT_CTRL1[4-13]");
         CME_PUTSCOM(C_CPLT_CTRL1_OR, core, BITS64(4, 11));
-        /// @todo add DD1 attribute control
-        PK_TRACE("DD1 only: reset sdis_n(flushing LCBES condition workaround");
+
+        PK_TRACE("Drop sdis_n(flushing LCBES condition) via CPLT_CONF0[34]");
         CME_PUTSCOM(C_CPLT_CONF0_CLEAR, core, BIT64(34));
+
         /// @todo add VDM_ENABLE attribute control
         PK_TRACE("Drop vdm enable via CPPM_VDMCR[0]");
         CME_PUTSCOM(PPM_VDMCR_CLR, core, BIT64(0));
 
-        PK_TRACE("SE2: Clock Sync Dropped");
+        PK_TRACE("Copy PECE CME sample to PPM Shadow via PECES");
 
-        // Copy PECE CME sample to PPM Shadow
         if (core & CME_MASK_C0)
         {
             scom_data = in64(CME_LCL_PECESR0);
@@ -373,12 +422,13 @@ p9_cme_stop_entry()
             G_cme_stop_record.act_level_c1 = STOP_LEVEL_2;
         }
 
+        PK_TRACE_INF("SE2.D: Clock Sync Dropped");
+
         //===========================
         MARK_TAG(SE_STOP2_DONE, core)
         //===========================
 
-        PK_TRACE("SE2.q");
-        // Update Stop History: In Core Stop Level 2
+        PK_TRACE("Update STOP history: in core stop level 2");
         // Check if STOP level 2 reaches the target for both or one core
         entry_ongoing =
             target_level == STOP_LEVEL_2 ?
@@ -472,8 +522,8 @@ p9_cme_stop_entry()
             //========================
         }
 
-        PK_TRACE("core[%d] running[%d] core_catchup[%d] origin_core[%d]",
-                 core, G_cme_stop_record.core_running, core_catchup, origin_core);
+        PK_TRACE_INF("Catch: core[%d] running[%d] core_catchup[%d] origin_core[%d]",
+                     core, G_cme_stop_record.core_running, core_catchup, origin_core);
 
 #endif
 
@@ -492,6 +542,7 @@ p9_cme_stop_entry()
         //===========================
         MARK_TRAP(SE_IS0_BEGIN)
         //===========================
+
 #if !SKIP_ABORT
         out32(CME_LCL_EIMR_CLR, (core << SHIFT32(13)) |
               (core << SHIFT32(15)) |
@@ -501,6 +552,7 @@ p9_cme_stop_entry()
               (core << SHIFT32(15)) |
               (core << SHIFT32(17)));
 #endif
+
         //===================
         MARK_TRAP(SE_IS0_END)
         //===================
@@ -508,8 +560,8 @@ p9_cme_stop_entry()
         core_aborted = core &  G_cme_stop_record.core_running;
         core         = core & ~G_cme_stop_record.core_running;
 
-        PK_TRACE("core[%d] running[%d] core_aborted[%d]",
-                 core, G_cme_stop_record.core_running, core_aborted);
+        PK_TRACE_INF("Abort: core[%d] running[%d] core_aborted[%d]",
+                     core, G_cme_stop_record.core_running, core_aborted);
 
         if (!core)
         {
@@ -528,12 +580,11 @@ p9_cme_stop_entry()
             deeper_core  = 0;
         }
 
-        PK_TRACE("SE2+:core[%d],deeper_core[%d],\
-                  target_level[%d],deeper_level[%d]",
-                 core, deeper_core, target_level, deeper_level);
+        PK_TRACE_INF("Check: core[%d] deeper_core[%d] target_level[%d] deeper_level[%d]",
+                     core, deeper_core, target_level, deeper_level);
 
         //----------------------------------------------------------------------
-        // STOP LEVEL 3
+        PK_TRACE_INF("+++++ +++++ STOP LEVEL 3 ENTRY +++++ +++++");
         //----------------------------------------------------------------------
 
         if (target_level == 3)
@@ -608,34 +659,34 @@ p9_cme_stop_entry()
         }
 
         //----------------------------------------------------------------------
-        // STOP LEVEL 4
+        PK_TRACE_INF("+++++ +++++ STOP LEVEL 4 ENTRY +++++ +++++");
         //----------------------------------------------------------------------
 
         //===============================
         MARK_TAG(SE_POWER_OFF_CORE, core)
         //===============================
 
-        // DD: Assert Cores Vital Thold/PCB Fence/Electrical Fence
-        PK_TRACE("SE4.a");
+        PK_TRACE("Assert PCB fence via NET_CTRL0[25]");
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
+
+        PK_TRACE("Assert electrical fence via NET_CTRL0[26]");
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(26));
+
+        PK_TRACE("Assert vital thold via NET_CTRL0[16]");
         CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(16));
 
 #if !STOP_PRIME
 
-        // Prepare PFET Controls
+        PK_TRACE("Drop vdd_pfet_val/sel_override/regulation_finger_en via PFCS[4,5,8]");
         // vdd_pfet_val/sel_override     = 0 (disbaled)
         // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
-        PK_TRACE("SE4.c");
         CME_PUTSCOM(PPM_PFCS_CLR, core, BIT64(4) | BIT64(5) | BIT64(8));
 
-        // Power Off Core VDD
+        PK_TRACE("Power off core VDD via PFCS[0-1]");
         // vdd_pfet_force_state = 01 (Force Voff)
-        PK_TRACE("SE4.d");
         CME_PUTSCOM(PPM_PFCS_OR, core, BIT64(1));
 
-        // Poll for power gate sequencer state: 0x8 (FSM Idle)
-        PK_TRACE("SE4.e");
+        PK_TRACE("Poll for power gate sequencer state: 0x8 (FSM Idle) via PFCS[42]");
 
         do
         {
@@ -643,15 +694,12 @@ p9_cme_stop_entry()
         }
         while(!(scom_data & BIT64(42)));
 
-        // Turn Off Force Voff
+        PK_TRACE("Turn off force voff via PFCS[0-1]");
         // vdd_pfet_force_state = 00 (Nop)
-        PK_TRACE("SE4.g");
         CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
 
 #endif
 
-        PK_TRACE("SE4: Core Powered Off");
-
         if (core & CME_MASK_C0)
         {
             G_cme_stop_record.act_level_c0 = STOP_LEVEL_4;
@@ -662,12 +710,13 @@ p9_cme_stop_entry()
             G_cme_stop_record.act_level_c1 = STOP_LEVEL_4;
         }
 
+        PK_TRACE_INF("SE4.A: Core Powered Off");
+
         //===========================
         MARK_TAG(SE_STOP4_DONE, core)
         //===========================
 
-        PK_TRACE("SE4.h");
-        // Update Stop History: In Core Stop Level 4
+        PK_TRACE("Update STOP history: in core stop level 4");
         // Check if STOP level 4 reaches the target for both or one core
         entry_ongoing =
             target_level == STOP_LEVEL_4 ? STOP_TRANS_COMPLETE :
@@ -709,6 +758,7 @@ p9_cme_stop_entry()
         //===========================
         MARK_TRAP(SE_IS1_BEGIN)
         //===========================
+
 #if !SKIP_ABORT
         out32(CME_LCL_EIMR_CLR, (core << SHIFT32(13)) |
               (core << SHIFT32(15)) |
@@ -718,6 +768,7 @@ p9_cme_stop_entry()
               (core << SHIFT32(15)) |
               (core << SHIFT32(17)));
 #endif
+
         //===================
         MARK_TRAP(SE_IS1_END)
         //===================
@@ -725,8 +776,8 @@ p9_cme_stop_entry()
         core_aborted = core &  G_cme_stop_record.core_running;
         core         = core & ~G_cme_stop_record.core_running;
 
-        PK_TRACE("core[%d] running[%d] core_aborted[%d]",
-                 core, G_cme_stop_record.core_running, core_aborted);
+        PK_TRACE_INF("Abort: core[%d] running[%d] core_aborted[%d]",
+                     core, G_cme_stop_record.core_running, core_aborted);
 
         if (!core)
         {
@@ -742,15 +793,14 @@ p9_cme_stop_entry()
                 target_level = deeper_level;
             }
 
-            deeper_core  = 0;
+            deeper_core = 0;
         }
 
-        PK_TRACE("SE4+:core[%d],deeper_core[%d],\
-                  target_level[%d],deeper_level[%d]",
-                 core, deeper_core, target_level, deeper_level);
+        PK_TRACE_INF("Check: core[%d] deeper_core[%d] target_level[%d] deeper_level[%d]",
+                     core, deeper_core, target_level, deeper_level);
 
         //----------------------------------------------------------------------
-        // STOP LEVEL 5 (preparsion of STOP LEVEL 8 and above)
+        PK_TRACE_INF("+++++ +++++ STOP LEVEL 5-7 ENTRY +++++ +++++");
         //----------------------------------------------------------------------
 
         if ((G_cme_stop_record.req_level_c0 >= STOP_LEVEL_8) &&
@@ -761,21 +811,20 @@ p9_cme_stop_entry()
             MARK_TAG(SE_PURGE_L2, CME_MASK_BC)
             //================================
 
-            // Assert L2+NCU Purge and NCU tlbie quiesce
-            // (chtm purge will be done in SGPE),
+            PK_TRACE("Assert L2+NCU purge and NCU tlbie quiesce via SICR[18,21,22]");
             // insert tlbie quiesce before ncu purge to avoid window condition
             // of ncu traffic still happening when purging starts
-            PK_TRACE("SE5.1a");
+            // Note: chtm purge will be done in SGPE
             out32(CME_LCL_SICR_OR, BIT32(18) | BIT32(21));
             out32(CME_LCL_SICR_OR, BIT32(22));
 
             // todo: poll for tlbie quiesce done?
 
-            // Poll for Purged Done
-            PK_TRACE("SE5.1b");
+            PK_TRACE("Poll for purged done via EISR[22,23]");
 
             do
             {
+
 #if !SKIP_L2_PURGE_ABORT
 
                 if (!core_aborted &&
@@ -793,27 +842,27 @@ p9_cme_stop_entry()
                         core_aborted |= CME_MASK_C1;
                     }
 
-                    PK_TRACE("L2 Purge aborted by core[%d]", core_aborted);
                     //=======================================
                     MARK_TAG(SE_PURGE_L2_ABORT, core_aborted)
                     //=======================================
-                    // abort L2+NCU purges
+
+                    PK_TRACE_INF("Abort: L2+NCU purge aborted by core[%d]", core_aborted);
                     out32(CME_LCL_SICR_OR, BIT32(19) | BIT32(23));
                 }
 
 #endif
+
             }
             while((in32(CME_LCL_EISR) & BITS32(22, 2)) != BITS32(22, 2));
 
-            // Deassert L2+NCU Purges, their possible aborts
-            PK_TRACE("SE5.1c");
+            PK_TRACE("Drop L2+NCU purges and their possible aborts via SICR[18,19,22,23]");
             out32(CME_LCL_SICR_CLR, (BITS32(18, 2) | BITS32(22, 2)));
 
-            PK_TRACE("SE5.1: L2/NCU/CHTM Purged");
+            PK_TRACE_INF("SE5.A: L2 and NCU Purged");
 
-            //===============================================================
+            //===================================================================
             MARK_TAG(SE_PURGE_L2_DONE, core_aborted ? core_aborted : CME_MASK_BC)
-            //===============================================================
+            //===================================================================
 
             // if core = 3 aborted = 1, core = 2(sgpe handoff) aborted (cme wakeup)
             // if core = 1 aborted = 1, core = 0(break)        aborted (cme wakeup)
@@ -832,8 +881,7 @@ p9_cme_stop_entry()
         MARK_TAG(SE_SGPE_HANDOFF, core)
         //=============================
 
-        // Update Stop History: In Core Stop Level 5
-        PK_TRACE("SE5.2a");
+        PK_TRACE("Update STOP history: in core stop level 5");
         CME_STOP_UPDATE_HISTORY(core,
                                 STOP_CORE_IS_GATED,
                                 STOP_TRANS_CORE_PORTION,
@@ -842,8 +890,7 @@ p9_cme_stop_entry()
                                 STOP_REQ_DISABLE,
                                 STOP_ACT_ENABLE);
 
-        // Send PCB Interrupt per core
-        PK_TRACE("SE5.2b");
+        PK_TRACE("Send PCB interrupt per core via PIG, select irq type via CPMMR[10]");
 
         if (core & CME_MASK_C0)
         {
@@ -883,22 +930,18 @@ p9_cme_stop_entry()
             G_cme_stop_record.act_level_c1 = STOP_LEVEL_5;
         }
 
-        // Change PPM Wakeup to STOPGPE
-        PK_TRACE("SE5.2c");
+        PK_TRACE("Switch PPM wakeup to STOP-GPE via CPMMR[13]");
         CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(13));
 
-        PK_TRACE("SE5.2: Handed off to SGPE");
+        PK_TRACE_INF("SE5.2B: Handed off to SGPE");
 
     }
     while(0);
 
     //--------------------------------------------------------------------------
-    // END OF STOP ENTRY
+    PK_TRACE_INF("+++++ +++++ END OF STOP ENTRY +++++ +++++");
     //--------------------------------------------------------------------------
 
-    // Release PPM Write Protection
-    CME_PUTSCOM(CPPM_CPMMR_CLR, CME_MASK_BC, BIT64(0));
-
     //============================
     MARK_TRAP(ENDSCOPE_STOP_ENTRY)
     //============================
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 4607518f..b4a61034 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -59,6 +59,7 @@ extern "C" {
 #define EQ_SCAN_REGION_TYPE      0x10030005
 #define EQ_CLK_REGION            0x10030006
 #define EQ_CLOCK_STAT_SL         0x10030008
+#define EQ_CLOCK_STAT_ARY        0x1003000A
 #define EQ_THERM_MODE_REG        0x1005000F
 
 #define EQ_BIST                  0x100F000B
@@ -66,6 +67,7 @@ extern "C" {
 #define EQ_NET_CTRL0_WAND        0x100F0041
 #define EQ_NET_CTRL0_WOR         0x100F0042
 #define C_NET_CTRL0              0x200F0040
+#define C_NET_CTRL0_WAND         0x200F0041
 #define C_NET_CTRL0_WOR          0x200F0042
 #define EQ_NET_CTRL1_WAND        0x100F0045
 #define EQ_NET_CTRL1_WOR         0x100F0046
@@ -93,6 +95,7 @@ extern "C" {
 #define EQ_QPPM_QCCR_WOR         0x100F01BF
 
 #define EX_NCU_STATUS_REG        0x1001100F
+#define EX_L3_MODE_REG1          0x1001180A
 #define EX_DRAM_REF_REG          0x1001180F
 #define EX_PM_PURGE_REG          0x10011813
 #define EX_PM_LCO_DIS_REG        0x10011816
@@ -111,6 +114,7 @@ extern "C" {
 #define PERV_OPCG_CAPT1          0x10030011
 #define PERV_OPCG_CAPT2          0x10030012
 #define PERV_CPLT_STAT0          0x10000100
+#define PERV_NET_CTRL1_WAND      0x000F0045
 
 /// Macro to update STOP History
 #define SGPE_STOP_UPDATE_HISTORY(id,base,gated,trans,req_l,act_l,req_e,act_e) \
@@ -175,7 +179,7 @@ enum SGPE_STOP_VECTOR_INDEX
     VECTOR_CONFIG                     = 2
 };
 
-#if HW386311_PBIE_RW_PTR_STOP11_FIX
+#if HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
 #define EXTRACT_RING_BITS(mask, ring, save) save = (ring) & (mask);
 #define RESTORE_RING_BITS(mask, ring, save) ring = (((ring) & (~mask)) | (save));
 #endif
@@ -196,12 +200,12 @@ typedef struct
 
 typedef struct
 {
-    uint32_t core[3];
-    uint32_t ex_l[3];
-    uint32_t ex_r[3];
-    uint32_t ex_b[3];
-    uint32_t quad[3];
-    uint32_t qswu[3];
+    uint32_t core[3]; // 24 bits
+    uint32_t ex_l[3]; // 6 bits
+    uint32_t ex_r[3]; // 6 bits
+    uint32_t ex_b[3]; // 12 bits
+    uint32_t quad[3]; // 6 bits
+    uint32_t qswu[3]; // 6 bits
 } sgpe_group_t;
 
 /// SGPE Stop Score Board Structure
@@ -228,14 +232,18 @@ int  p9_sgpe_stop_exit();
 int  p9_hcd_cache_scan0(uint32_t, uint64_t, uint64_t);
 int  p9_hcd_cache_poweron(uint32_t);
 int  p9_hcd_cache_chiplet_reset(uint32_t, uint32_t);
+int  p9_hcd_cache_chiplet_l3_dcc_setup(uint32_t);
 int  p9_hcd_cache_gptr_time_initf(uint32_t);
+int  p9_hcd_cache_dpll_initf(uint32_t);
 int  p9_hcd_cache_dpll_setup(uint32_t);
+int  p9_hcd_cache_dcc_skewadjust_setup(uint32_t);
 int  p9_hcd_cache_chiplet_init(uint32_t);
 int  p9_hcd_cache_repair_initf(uint32_t);
 int  p9_hcd_cache_arrayinit(uint32_t, uint32_t ex);
+int  p9_hcd_cache_initf(uint32_t);
 int  p9_hcd_cache_startclocks(uint32_t, uint32_t);
 int  p9_hcd_cache_l2_startclocks(uint32_t, uint32_t, uint32_t);
-int  p9_hcd_cache_scominit(uint32_t);
+int  p9_hcd_cache_scominit(uint32_t, uint32_t);
 int  p9_hcd_cache_scomcust(uint32_t);
 int  p9_hcd_cache_ras_runtime_scom(uint32_t);
 int  p9_hcd_cache_occ_runtime_scom(uint32_t);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index 187ad5fe..ebbbeed3 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -27,7 +27,9 @@
 #include "p9_sgpe_stop_enter_marks.h"
 
 extern SgpeStopRecord G_sgpe_stop_record;
-#if HW386311_PBIE_RW_PTR_STOP11_FIX
+
+#if HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
+
 uint64_t G_ring_save[MAX_QUADS][8] =
 {
     {0, 0, 0, 0, 0, 0, 0, 0},
@@ -37,6 +39,7 @@ uint64_t G_ring_save[MAX_QUADS][8] =
     {0, 0, 0, 0, 0, 0, 0, 0},
     {0, 0, 0, 0, 0, 0, 0, 0}
 };
+
 uint64_t G_ring_spin[10][2] =
 {
     {0,    0},
@@ -50,6 +53,7 @@ uint64_t G_ring_spin[10][2] =
     {6343, 0xC1E061FFED5F0000}, //29
     {17871, 0}                  //128
 };
+
 #endif
 
 int
@@ -67,10 +71,14 @@ p9_sgpe_stop_entry()
     uint64_t     scom_data;
     uint64_t     temp_data;
     ppm_sshsrc_t hist;
-#if HW386311_PBIE_RW_PTR_STOP11_FIX
+#if HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
     int          spin;
 #endif
 
+    //--------------------------------------------------------------------------
+    PK_TRACE_INF("+++++ +++++ BEGIN OF STOP ENTRY +++++ +++++");
+    //--------------------------------------------------------------------------
+
     //================================
     MARK_TAG(BEGINSCOPE_STOP_ENTRY, 0)
     //================================
@@ -130,21 +138,21 @@ p9_sgpe_stop_entry()
         if (G_sgpe_stop_record.group.ex_b[VECTOR_ENTRY] ||
             G_sgpe_stop_record.group.quad[VECTOR_ENTRY])
         {
-            PK_TRACE("clv[%d][%d][%d][%d]",
-                     G_sgpe_stop_record.level[qloop][0],
-                     G_sgpe_stop_record.level[qloop][1],
-                     G_sgpe_stop_record.level[qloop][2],
-                     G_sgpe_stop_record.level[qloop][3]);
+            PK_TRACE_INF("Actual:  clv[%d][%d][%d][%d]",
+                         G_sgpe_stop_record.level[qloop][0],
+                         G_sgpe_stop_record.level[qloop][1],
+                         G_sgpe_stop_record.level[qloop][2],
+                         G_sgpe_stop_record.level[qloop][3]);
 
-            PK_TRACE("act: qlv[%d]x0lv[%d]x1lv[%d]",
-                     G_sgpe_stop_record.state[qloop].act_state_q,
-                     G_sgpe_stop_record.state[qloop].act_state_x0,
-                     G_sgpe_stop_record.state[qloop].act_state_x1);
+            PK_TRACE_INF("Actual:  qlv:[%d]x0lv[%d]x1lv[%d]",
+                         G_sgpe_stop_record.state[qloop].act_state_q,
+                         G_sgpe_stop_record.state[qloop].act_state_x0,
+                         G_sgpe_stop_record.state[qloop].act_state_x1);
 
-            PK_TRACE("req: qlv[%d]x0lv[%d]x1lv[%d]",
-                     G_sgpe_stop_record.state[qloop].req_state_q,
-                     G_sgpe_stop_record.state[qloop].req_state_x0,
-                     G_sgpe_stop_record.state[qloop].req_state_x1);
+            PK_TRACE_INF("Request: qlv[%d]x0lv[%d]x1lv[%d]",
+                         G_sgpe_stop_record.state[qloop].req_state_q,
+                         G_sgpe_stop_record.state[qloop].req_state_x0,
+                         G_sgpe_stop_record.state[qloop].req_state_x1);
         }
     }
 
@@ -157,11 +165,11 @@ p9_sgpe_stop_entry()
     G_sgpe_stop_record.group.quad[VECTOR_ENTRY] &=
         G_sgpe_stop_record.group.quad[VECTOR_CONFIG];
 
-    PK_TRACE("Core Entry Vectors:   X[%x] X0[%x] X1[%x] Q[%x]",
-             G_sgpe_stop_record.group.ex_b[VECTOR_ENTRY],
-             G_sgpe_stop_record.group.ex_l[VECTOR_ENTRY],
-             G_sgpe_stop_record.group.ex_r[VECTOR_ENTRY],
-             G_sgpe_stop_record.group.quad[VECTOR_ENTRY]);
+    PK_TRACE_INF("Core Entry Vectors:   X[%x] X0[%x] X1[%x] Q[%x]",
+                 G_sgpe_stop_record.group.ex_b[VECTOR_ENTRY],
+                 G_sgpe_stop_record.group.ex_l[VECTOR_ENTRY],
+                 G_sgpe_stop_record.group.ex_r[VECTOR_ENTRY],
+                 G_sgpe_stop_record.group.quad[VECTOR_ENTRY]);
 
     //TODO: message pgpe to suspend Pstate only if stop level >= 8
     if (G_sgpe_stop_record.group.quad[VECTOR_ENTRY])
@@ -178,9 +186,7 @@ p9_sgpe_stop_entry()
     }
 
 
-    // ------------------------------------------------------------------------
-    // EX STOP ENTRY [LEVEL 8-10]
-    // ------------------------------------------------------------------------
+
     // only stop 8 sets x_in
     for(xentry = G_sgpe_stop_record.group.ex_b[VECTOR_ENTRY], qloop = 0;
         xentry > 0;
@@ -192,17 +198,29 @@ p9_sgpe_stop_entry()
             continue;
         }
 
-        PK_TRACE("q[%d]exmask[%d] starts entry", qloop, ex);
+        // ------------------------------------------------------------------------
+        PK_TRACE_INF("+++++ +++++ EX STOP ENTRY [LEVEL 8-10] +++++ +++++");
+        // ------------------------------------------------------------------------
 
-        // Update QSSR: stop_entry_ongoing
+        PK_TRACE_INF("Check: q[%d]ex[%d] start ex entry", qloop, ex);
+
+        PK_TRACE("Update QSSR: stop_entry_ongoing");
         out32(OCB_QSSR_OR, BIT32(qloop + 20));
 
-        // Update History for ongoing stop 8 entry
         cloop  = (ex & FST_EX_IN_QUAD) ?              0 : CORES_PER_EX;
         climit = (ex & SND_EX_IN_QUAD) ? CORES_PER_QUAD : CORES_PER_EX;
 
         for(; cloop < climit; cloop++)
         {
+            // Check partial good core
+            if (!(G_sgpe_stop_record.group.core[VECTOR_CONFIG] &
+                  BIT32(((qloop << 2) + cloop))))
+            {
+                continue;
+            }
+
+            PK_TRACE("Update STOP history on core[%d]: in transition of entry",
+                     ((qloop << 2) + cloop));
             SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
                                      CORE_ADDR_BASE,
                                      STOP_CORE_IS_GATED,
@@ -217,9 +235,8 @@ p9_sgpe_stop_entry()
         MARK_TAG(SE_STOP_L2_CLKS, ((ex << 6) | (32 >> qloop)))
         //====================================================
 
-        PK_TRACE("SE8.a");
+        PK_TRACE("Drop L2 Snoop(quiesce L2-L3 interface) via EX_PM_L2_RCMD_DIS_REG[0]");
 
-        // Disable L2 Snoop(quiesce L2-L3 interface, what about NCU?)
         if (ex & FST_EX_IN_QUAD)
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_L2_RCMD_DIS_REG, qloop, 0),
                         BIT64(0));
@@ -230,7 +247,7 @@ p9_sgpe_stop_entry()
 
         PPE_WAIT_CORE_CYCLES(loop, 256)
 
-        PK_TRACE("set partial bad l2/l3 and stopping/stoped l2 pscom masks");
+        PK_TRACE("Assert partial bad L2/L3 and stopping/stoped l2 pscom masks via RING_FENCE_MASK_LATCH");
         scom_data = 0;
 
         if (!(G_sgpe_stop_record.group.ex_l[VECTOR_CONFIG] & BIT32(qloop)))
@@ -255,18 +272,17 @@ p9_sgpe_stop_entry()
 
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_RING_FENCE_MASK_LATCH, qloop), scom_data);
 
-        PK_TRACE("SE8.b");
-        // Set all bits to zero prior stop cache clocks
+
+
+        PK_TRACE("Clear SCAN_REGION_TYPE prior to stop L2 clocks");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
 
-        PK_TRACE("SE8.c");
-        // Stop L2 Clocks
+        PK_TRACE("Stop L2 clocks via CLK_REGION[8/9]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop),
                     (CLK_STOP_CMD | CLK_THOLD_ALL |
                      ((uint64_t)ex << SHIFT64(9))));
 
-        PK_TRACE("SE8.d");
-        // Poll for L2 clocks stopped
+        PK_TRACE("Poll for L2 clocks stopped via CPLT_STAT0[8]");
 
         do
         {
@@ -274,44 +290,40 @@ p9_sgpe_stop_entry()
         }
         while((~scom_data) & BIT64(8));
 
+        PK_TRACE("Check L2 clock is stopped via CLOCK_STAT_SL[4-13]");
         GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
 
         if (((~scom_data) & ((uint64_t)ex << SHIFT64(9))) != 0)
         {
-            PK_TRACE("L2 clock stop failed");
+            PK_TRACE("ERROR: L2 clock stop failed. HALT SGPE!");
             pk_halt();
         }
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
 
-        PK_TRACE("SE8: L2 Clock Stopped");
+        PK_TRACE_INF("SE8.A: L2 Clock Stopped");
 
         //========================
         MARK_TRAP(SE_STOP_L2_GRID)
         //========================
 
-        PK_TRACE("SE8.e");
-        // Drop clock sync enable before switch to refclk
+        PK_TRACE("Drop clock sync enable before switch to refclk via EXCGCR[36/37]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_EXCGCR_CLR, qloop),
                     ((uint64_t)ex << SHIFT64(37)));
 
-        PK_TRACE("SE8.f");
+        PK_TRACE("Poll for clock sync done to drop via QACSR[36/37]");
 
-        // Poll for clock sync done to drop
         do
         {
             GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QACSR, qloop), scom_data);
         }
         while(((~scom_data >> SHIFT64(37)) & ex) != ex);
 
-        PK_TRACE("SE8.g");
-        // Switch glsmux to refclk to save clock grid power
+        PK_TRACE("Switch glsmux to refclk to save clock grid power via EXCGCR[34/35]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_EXCGCR_CLR, qloop),
                     ((uint64_t)ex << SHIFT64(35)));
 
-        PK_TRACE("SE8.h");
-
         if (ex & FST_EX_IN_QUAD)
         {
             cloop = 0;
@@ -340,8 +352,17 @@ p9_sgpe_stop_entry()
 
         for(; cloop < climit; cloop++)
         {
+            // Check partial good core
+            if (!(G_sgpe_stop_record.group.core[VECTOR_CONFIG] &
+                  BIT32(((qloop << 2) + cloop))))
+            {
+                continue;
+            }
+
             // request levle already set by CME
             // shift by 2 == times 4, which is cores per quad
+            PK_TRACE("Update STOP history on core[%d]: in stop level 8",
+                     ((qloop << 2) + cloop));
             SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
                                      CORE_ADDR_BASE,
                                      STOP_CORE_IS_GATED,
@@ -352,19 +373,19 @@ p9_sgpe_stop_entry()
                                      STOP_ACT_ENABLE);
         }
 
-        // Update QSSR: l2_stopped, drop stop_entry_ongoing
+        PK_TRACE("Update QSSR: l2_stopped, drop stop_entry_ongoing");
         out32(OCB_QSSR_CLR, BIT32(qloop + 20));
         out32(OCB_QSSR_OR, (ex << SHIFT32((qloop << 1) + 1)));
 
+        PK_TRACE_INF("SE8.B: L2 Clock Sync Dropped");
+
         //==================================================
         MARK_TAG(SE_STOP8_DONE, ((ex << 6) | (32 >> qloop)))
         //==================================================
 
     };
 
-    // ------------------------------------------------------------------------
-    // QUAD STOP ENTRY [LEVEL 11-15]
-    // ------------------------------------------------------------------------
+
 
     for(qentry = G_sgpe_stop_record.group.quad[VECTOR_ENTRY], qloop = 0, ex = 0;
         qentry > 0;
@@ -376,6 +397,10 @@ p9_sgpe_stop_entry()
             continue;
         }
 
+        // ------------------------------------------------------------------------
+        PK_TRACE("+++++ +++++ QUAD STOP ENTRY [LEVEL 11-15] +++++ +++++");
+        // ------------------------------------------------------------------------
+
         if (G_sgpe_stop_record.group.ex_l[VECTOR_CONFIG] & BIT32(qloop))
         {
             ex |= FST_EX_IN_QUAD;
@@ -386,12 +411,12 @@ p9_sgpe_stop_entry()
             ex |= SND_EX_IN_QUAD;
         }
 
-        PK_TRACE("q[%d]x[%d] starts quad entry", qloop, ex);
+        PK_TRACE_INF("Check: q[%d]ex[%d] starts quad entry", qloop, ex);
 
-        // Update QSSR: stop_entry_ongoing
+        PK_TRACE("Update QSSR: stop_entry_ongoing");
         out32(OCB_QSSR_OR, BIT32(qloop + 20));
 
-        // Update Quad STOP History
+        PK_TRACE("Update STOP history on quad[%d]: update request stop level", qloop);
         SGPE_STOP_UPDATE_HISTORY(qloop,
                                  QUAD_ADDR_BASE,
                                  STOP_CACHE_IS_GATED,
@@ -405,9 +430,8 @@ p9_sgpe_stop_entry()
         MARK_TAG(SE_PURGE_L3, (32 >> qloop))
         //==================================
 
-        PK_TRACE("SE11.a");
+        PK_TRACE("Drop LCO prior to purge via EX_PM_LCO_DIS_REG[0]");
 
-        // Disable LCO prior to purge
         if(ex & FST_EX_IN_QUAD)
         {
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG, qloop, 0),
@@ -421,9 +445,9 @@ p9_sgpe_stop_entry()
         }
 
 #if !SKIP_L3_PURGE
-        PK_TRACE("SE11.b");
 
-        // Assert Purge L3
+        PK_TRACE("Assert purge L3 via EX_PM_PURGE_REG[0]");
+
         if(ex & FST_EX_IN_QUAD)
         {
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, 0), BIT64(0));
@@ -436,21 +460,22 @@ p9_sgpe_stop_entry()
 
         // todo: stop debug trace, attribute may be needed
 
-        PK_TRACE("SE11.d");
+        PK_TRACE("Poll for L3 purge done via EX_PM_PURGE_REG[0]");
 
-        // Poll for purge done on the same request bit thus no need to deassert
+        // Poll on the same request bit thus no need to deassert
         do
         {
+
 #if !SKIP_L3_PURGE_ABORT
 
             if (in32(OCB_OISR1) & (BITS32(15, 2) | BIT32(19)))
             {
-                PK_TRACE("SE: interrupt detected");
+                PK_TRACE_INF("Abort: interrupt detected");
 
                 if ((in32(OCB_OPITNPRA(2)) & BITS32((qloop << 2), 4)) ||
                     (in32(OCB_OPITNPRA(3)) & BITS32((qloop << 2), 4)))
                 {
-                    PK_TRACE("SE: core interrupt detected");
+                    PK_TRACE_INF("Abort: core interrupt detected");
 
                     for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
                     {
@@ -459,7 +484,7 @@ p9_sgpe_stop_entry()
                             (in32(OCB_OPIT3CN(((qloop << 2) + cloop))) &
                              TYPE3_PAYLOAD_EXIT_EVENT))
                         {
-                            PK_TRACE("SE: core wakeup detected");
+                            PK_TRACE_INF("Abort: core wakeup detected");
                             l3_purge_aborted = 1;
                             break;
                         }
@@ -469,7 +494,7 @@ p9_sgpe_stop_entry()
                 if ((in32(OCB_OPIT6PRB) & BIT32(qloop)) &&
                     (in32(OCB_OPIT6QN(qloop)) & TYPE6_PAYLOAD_EXIT_EVENT))
                 {
-                    PK_TRACE("SE: quad wakeup detected");
+                    PK_TRACE_INF("Abort: quad wakeup detected");
                     l3_purge_aborted = 1;
                 }
 
@@ -480,7 +505,8 @@ p9_sgpe_stop_entry()
                     MARK_TAG(SE_PURGE_L3_ABORT, (32 >> qloop))
                     //========================================
 
-                    // Assert Purge L3 Abort
+                    PK_TRACE_INF("Abort: assert purge L3 abort");
+
                     if (ex & FST_EX_IN_QUAD)
                         GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, 0),
                                     BIT64(2));
@@ -489,7 +515,8 @@ p9_sgpe_stop_entry()
                         GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_PURGE_REG, qloop, 1),
                                     BIT64(2));
 
-                    // Poll for Abort Done
+                    PK_TRACE_INF("Abort: poll for abort done");
+
                     if(ex & FST_EX_IN_QUAD)
                     {
                         do
@@ -514,7 +541,8 @@ p9_sgpe_stop_entry()
                     MARK_TAG(SE_PURGE_L3_ABORT_DONE, (32 >> qloop))
                     //=============================================
 
-                    // Deassert LCO Disable
+                    PK_TRACE_INF("Abort: Drop LCO Disable");
+
                     if (ex & FST_EX_IN_QUAD)
                         GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG,
                                                      qloop, 0), 0);
@@ -523,7 +551,7 @@ p9_sgpe_stop_entry()
                         GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_PM_LCO_DIS_REG,
                                                      qloop, 1), 0);
 
-                    // Notify PGPE to resume
+                    // TODO Notify PGPE to resume
                 }
             }
 
@@ -547,44 +575,48 @@ p9_sgpe_stop_entry()
 
         if (l3_purge_aborted)
         {
+            PK_TRACE_INF("Abort: L3 Purge Aborted");
             continue;
         }
 
+        PK_TRACE_INF("SE11.A: L3 Purged");
+
 #endif
 
         //==================================
         MARK_TAG(SE_PURGE_PB, (32 >> qloop))
         //==================================
 
-        PK_TRACE("SE11.g");
-        // Assert PowerBus purge
+        PK_TRACE("Assert powerbus purge via QCCR[30]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WOR, qloop), BIT64(30));
 
-        PK_TRACE("SE11.h");
+        PK_TRACE("Poll PowerBus purge done via QCCR[31]");
 
-        // Poll PowerBus purge done
         do
         {
             GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR, qloop), scom_data);
         }
         while(!(scom_data & BIT64(31)));
 
-        PK_TRACE("SE11.i");
-        // Deassert PowerBus purge
+        PK_TRACE("Drop powerbus purge via QCCR[30]");
         // todo may need to move this to wakeup
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_QPPM_QCCR_WCLEAR, qloop), BIT64(30));
 
+        PK_TRACE_INF("SE11.B: PowerBus Purged");
+
         //===========================================
         MARK_TAG(SE_WAIT_PGPE_SUSPEND, (32 >> qloop))
         //===========================================
+
         // TODO: Poll PGPE Suspend Ack
 
         //======================================
         MARK_TAG(SE_QUIESCE_QUAD, (32 >> qloop))
         //======================================
 
-        // todo halt cme here
-        PK_TRACE("SE11.j");
+        // TODO halt cme here
+
+        PK_TRACE("Assert refresh quiesce prior to L3 (refresh domain) stop clk via EX_DRAM_REF_REG[7]");
 
         // Assert refresh quiesce prior to L3 (refresh domain) stop clk
         // Edram quiesce is asserted by hardware when l3 thold is asserted in cc
@@ -602,7 +634,8 @@ p9_sgpe_stop_entry()
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_DRAM_REF_REG, qloop, 1), scom_data);
         }
 
-        // Check NCU_SATUS_REG[0:3] for all zeros
+        PK_TRACE("Check NCU_SATUS_REG[0:3] for all zeros");
+
         if (ex & FST_EX_IN_QUAD)
         {
             do
@@ -623,29 +656,26 @@ p9_sgpe_stop_entry()
             while((~scom_data & BITS64(0, 4)) != BITS64(0 , 4));
         }
 
+        PK_TRACE_INF("SE11.C: NCU Status Clean");
+
         //===========================
         MARK_TRAP(SE_STOP_CACHE_CLKS)
         //===========================
 
-        PK_TRACE("SE11.k");
-        // Raise Cache Logical fence
+        PK_TRACE("Assert cache chiplet fence via NET_CTRL0[18]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(18));
 
-        // clock off cache chiplet
-        PK_TRACE("SE11.l");
-        // Set all bits to zero prior stop core clocks
+        PK_TRACE("Clear SCAN_REGION prior to stop cache clocks");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
 
-        PK_TRACE("SE11.m");
-        // Stop Cache Clocks
+        PK_TRACE("Stop cache clocks via CLK_REGION");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLK_REGION, qloop),
                     (CLK_STOP_CMD | CLK_THOLD_ALL |
                      CLK_REGION_ALL_BUT_EX        |
                      ((uint64_t)ex << SHIFT64(7)) |
                      ((uint64_t)ex << SHIFT64(13))));
 
-        PK_TRACE("SE11.n");
-        // Poll for Cache clocks stopped
+        PK_TRACE("Poll for cache clocks stopped via CPLT_STAT0[8]");
 
         do
         {
@@ -653,27 +683,27 @@ p9_sgpe_stop_entry()
         }
         while((~scom_data) & BIT64(8));
 
+        PK_TRACE("Check core clock is stopped via CLOCK_STAT_SL[4-13]");
         GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_CLOCK_STAT_SL, qloop), scom_data);
 
         if (((~scom_data) & (CLK_REGION_ALL_BUT_EX        |
                              ((uint64_t)ex << SHIFT64(7)) |
                              ((uint64_t)ex << SHIFT64(13)))) != 0)
         {
-            PK_TRACE("Cache clock stop failed");
+            PK_TRACE("ERROR: Cache clock stop failed. HALT SGPE!");
             pk_halt();
         }
 
         // MF: verify compiler generate single rlwmni
         // MF: delay may be needed for stage latch to propagate thold
 
-        PK_TRACE("SE11: Cache Clock Stopped");
-
-        PK_TRACE("SE11.o");
-        // Switch glsmux to refclk to save clock grid power
+        PK_TRACE("Switch glsmux to refclk to save clock grid power via CGCR[3]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), 0);
-        // Assert Vital Fence
+
+        PK_TRACE("Assert vital fence via CPLT_CTRL1[3]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop), BIT64(3));
-        // Raise Partial Good Fences
+
+        PK_TRACE("Assert partial good regional fences via CPLT_CTRL1[4-14]");
         // Must cover partial bad fences as well or powerbus error will raise
         // Note: Stop11 will lose all the fences so here needs to assert them
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop),
@@ -681,30 +711,36 @@ p9_sgpe_stop_entry()
                      ((uint64_t)ex << SHIFT64(7)) |
                      ((uint64_t)ex << SHIFT64(9)) |
                      ((uint64_t)ex << SHIFT64(13))));
+
         /// @todo add VDM_ENABLE attribute control
-        PK_TRACE("Assert vdm enable via CPPM_VDMCR[0]");
+        PK_TRACE("Drop vdm enable via CPPM_VDMCR[0]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_VDMCR_CLR, qloop), BIT64(0));
 
+        PK_TRACE_INF("SE11.D: Cache Clock Stopped");
+
         //=========================================
         MARK_TAG(SE_POWER_OFF_CACHE, (32 >> qloop))
         //=========================================
 
-#if HW386311_PBIE_RW_PTR_STOP11_FIX
+#if HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
+
+        PK_TRACE_INF("FCMS: Engage with PBIE Read/Write Pointer Scan Workaround");
+
         // bit4,5,11 = perv/eqpb/pbieq, bit59 = inex
-        PK_TRACE("SE: Setup scan register to select the ring");
+        PK_TRACE("FCMS: Setup scan register to select the ring");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(0x10030005, qloop), BITS64(4, 2) | BIT64(11) | BIT64(59));
 
-        PK_TRACE("SE: checkword set");
+        PK_TRACE("FCMS: checkword set");
         scom_data = 0xa5a5a5a5a5a5a5a5;
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(0x1003E000, qloop), scom_data);
 
         for(spin = 1;; spin++)
         {
-            PK_TRACE("SE: spin ring loop%d", spin);
+            PK_TRACE("FCMS: spin ring loop%d", spin);
             scom_data = (G_ring_spin[spin][0] - G_ring_spin[spin - 1][0]) << 32;
             GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(0x10039000, qloop), scom_data);
 
-            PK_TRACE("SE: Poll OPCG done for ring spin");
+            PK_TRACE("FCMS: Poll OPCG done for ring spin");
 
             do
             {
@@ -714,47 +750,51 @@ p9_sgpe_stop_entry()
 
             if (spin == 9)
             {
-                PK_TRACE("SE: checkword check");
+                PK_TRACE("FCMS: checkword check");
                 GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(0x1003E000, qloop), scom_data);
 
                 if (scom_data != 0xa5a5a5a5a5a5a5a5)
                 {
-                    PK_TRACE("checkword[%x%x] failed", UPPER32(scom_data), LOWER32(scom_data));
+                    PK_TRACE("ERROR: checkword[%x%x] failed. HALT SGPE!",
+                             UPPER32(scom_data), LOWER32(scom_data));
                     pk_halt();
                 }
 
                 break;
             }
 
-            PK_TRACE("SE: save pbie read ptr");
+            PK_TRACE("FCMS: save pbie read ptr");
             GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(0x1003E000, qloop), scom_data);
             EXTRACT_RING_BITS(G_ring_spin[spin][1], scom_data, G_ring_save[qloop][spin - 1]);
-            PK_TRACE("SE: mask: %8x %8x",
+            PK_TRACE("FCMS: mask: %8x %8x",
                      UPPER32(G_ring_spin[spin][1]),
                      LOWER32(G_ring_spin[spin][1]));
-            PK_TRACE("SE: ring: %8x %8x",
+            PK_TRACE("FCMS: ring: %8x %8x",
                      UPPER32(scom_data),
                      LOWER32(scom_data));
-            PK_TRACE("SE: save: %8x %8x",
+            PK_TRACE("FCMS: save: %8x %8x",
                      UPPER32(G_ring_save[qloop][spin - 1]),
                      LOWER32(G_ring_save[qloop][spin - 1]));
         }
 
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(0x10030005, qloop), 0);
+
 #endif
 
-        // DD: Assert Cache Vital Thold/PCB Fence/Electrical Fence
-        PK_TRACE("SE11.q");
+        PK_TRACE("Assert PCB fence via NET_CTRL0[25]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(25));
+
+        PK_TRACE("Assert electrical fence via NET_CTRL0[26]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(26));
+
+        PK_TRACE("Assert vital thold via NET_CTRL0[16]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(16));
 
-        // L3 edram shutdown
+        PK_TRACE("Shutdown L3[%d] EDRAM via QCCR[0-3/4-7]", ex);
         // QCCR[0/4] EDRAM_ENABLE_DC
         // QCCR[1/5] EDRAM_VWL_ENABLE_DC
         // QCCR[2/6] L3_EX0/1_EDRAM_VROW_VBLH_ENABLE_DC
         // QCCR[3/7] EDRAM_VPP_ENABLE_DC
-        PK_TRACE("SE11.r");
 
         if (ex & SND_EX_IN_QUAD)
         {
@@ -781,23 +821,20 @@ p9_sgpe_stop_entry()
         }
 
 #if !STOP_PRIME
-        // todo needed?
-        // Prepare PFET Controls
+
+        PK_TRACE("Drop vdd/vcs_pfet_val/sel_override/regulation_finger_en via PFCS[4-7,8]");
         // vdd_pfet_val/sel_override     = 0 (disbaled)
         // vcs_pfet_val/sel_override     = 0 (disbaled)
         // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
-        PK_TRACE("SE11.t");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, qloop),
                     BITS64(4, 4) | BIT64(8));
 
-        // Power Off Cache VDS then VDD
+        PK_TRACE("Power off VCS via PFCS[2-3]");
         // vcs_pfet_force_state = 01 (Force Voff)
-        PK_TRACE("SE11.u");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, qloop), BIT64(3));
 
+        PK_TRACE("Poll for power gate sequencer state: 0x8 (FSM Idle) via PFCS[50]");
         // todo: poll the sense line instead
-        // Poll for power gate sequencer state: 0x8 (FSM Idle)
-        PK_TRACE("SE11.v");
 
         do
         {
@@ -805,13 +842,12 @@ p9_sgpe_stop_entry()
         }
         while(!(scom_data & BIT64(50)));
 
+        PK_TRACE("Power off VDD via PFCS[0-1]");
         // vdd_pfet_force_state = 01 (Force Voff)
-        PK_TRACE("SE11.u");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_OR, qloop), BIT64(1));
 
+        PK_TRACE("Poll for power gate sequencer state: 0x8 (FSM Idle) via PFCS[42]");
         // todo: poll the sense line instead
-        // Poll for power gate sequencer state: 0x8 (FSM Idle)
-        PK_TRACE("SE11.v");
 
         do
         {
@@ -819,19 +855,31 @@ p9_sgpe_stop_entry()
         }
         while(!(scom_data & BIT64(42)));
 
-        // Turn Off Force Voff
+        PK_TRACE("Turn off force voff via PFCS[0-3]");
         // vdd_pfet_force_state = 00 (Nop)
         // vcs_pfet_force_state = 00 (Nop)
-        PK_TRACE("SE11.x");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_PFCS_CLR, qloop), BITS64(0, 4));
 
+        PK_TRACE_INF("SE11.E: Cache Powered Off");
+
 #endif
 
-        PK_TRACE("SE11.y");
+        PK_TRACE("Drop CME_INTERPPM_DPLL_ENABLE after DPLL is stopped via QPMMR[26]");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, qloop), BIT64(20) | BIT64(22) | BIT64(24) | BIT64(26));
+
         G_sgpe_stop_record.state[qloop].act_state_q = STOP_LEVEL_11;
 
         for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
         {
+            // Check partial good core
+            if (!(G_sgpe_stop_record.group.core[VECTOR_CONFIG] &
+                  BIT32(((qloop << 2) + cloop))))
+            {
+                continue;
+            }
+
+            PK_TRACE("Update STOP history on core[%d]: in stop level 11",
+                     ((qloop << 2) + cloop));
             SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
                                      CORE_ADDR_BASE,
                                      STOP_CORE_IS_GATED,
@@ -842,6 +890,7 @@ p9_sgpe_stop_entry()
                                      STOP_ACT_ENABLE);
         }
 
+        PK_TRACE("Update STOP history on quad[%d]: in stop level 11", qloop);
         SGPE_STOP_UPDATE_HISTORY(qloop,
                                  QUAD_ADDR_BASE,
                                  STOP_CACHE_IS_GATED,
@@ -851,10 +900,10 @@ p9_sgpe_stop_entry()
                                  STOP_REQ_DISABLE,
                                  STOP_ACT_ENABLE);
 
-        // Update QSSR: quad_stopped
+        PK_TRACE("Update QSSR: quad_stopped");
         out32(OCB_QSSR_OR, BIT32(qloop + 14));
 
-        // Update QSSR: drop stop_entry_ongoing
+        PK_TRACE("Update QSSR: drop stop_entry_ongoing");
         out32(OCB_QSSR_CLR, BIT32(qloop + 20));
 
         //=====================================
@@ -862,7 +911,11 @@ p9_sgpe_stop_entry()
         //=====================================
     }
 
-    //loop quad to drop spwu done + clear qswu
+    //--------------------------------------------------------------------------
+    PK_TRACE_INF("+++++ +++++ END OF STOP ENTRY +++++ +++++");
+    //--------------------------------------------------------------------------
+
+    //loop quad to clear qswu record
     for(qloop = 0; qloop < MAX_QUADS; qloop++)
     {
         if (G_sgpe_stop_record.group.qswu[VECTOR_ENTRY] & BIT32(qloop))
@@ -874,5 +927,6 @@ p9_sgpe_stop_entry()
     //============================
     MARK_TRAP(ENDSCOPE_STOP_ENTRY)
     //============================
+
     return SGPE_STOP_SUCCESS;
 }