CME/SGPE: update stop cme/sgpe images

Change-Id: I9c2de4e90e7445d123290de37fff9e26dae2d1ae
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/22464
Tested-by: Jenkins Server
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

13 files changed, 225 insertions, 258 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
index 92512c7e..41da93b7 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
@@ -178,6 +178,6 @@ void pk_unified_irq_prty_mask_handler(void)
     l_vecL = 0;
     asm volatile ("lvd %[data], 0(%[addr]) \n" \
                   : [data]"=r"(l_vecH) \
-                  : [addr]"r"(&ext_irq_vectors_cme[iPrtyLvl][IDX_PRTY_VEC]) );
+                  : [addr]"r"(&ext_irq_vector_pk) );
 
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
index 0bb06884..0f478706 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
@@ -34,7 +34,6 @@
 /// \brief Application specific overrides go here.
 ///
 
-#define EPM_P9_LEVEL        1
 #define STOP_PRIME          0
 #define SKIP_ABORT          0
 #define SKIP_L2_PURGE_ABORT 0
@@ -44,13 +43,18 @@
 #define SKIP_INITF          0
 #define SKIP_ARY_INIT       0
 #define SKIP_SELF_RESTORE   0
+#define SKIP_BCE            1
 
 // --------------------
 
 #define EPM_P9_TUNING         1
-#define SIMICS_TUNING         1
-#define DEV_DEBUG             1
+#define SIMICS_TUNING         0
 #define USE_SIMICS_IO         0
+#define DEV_DEBUG             1
+
+#if EPM_P9_TUNING
+    #define PK_TRACE_BUFFER_WRAP_MARKER 1
+#endif
 #define PK_TRACE_TIMER_OUTPUT 0
 
 // --------------------
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
index 2378fbed..e2f1bdb4 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
@@ -44,18 +44,7 @@
 #include "cppm_firmware_registers.h"
 
 #include "p9_stop_common.h"
-/*
-/// Compiler flags
-#define EPM_P9_LEVEL        1
-#define SKIP_ABORT          1
-#define SKIP_L2_PURGE_ABORT 1
-#define SKIP_CATCHUP        1
-#define STOP_PRIME          0
-#define SKIP_SCAN0          0
-#define SKIP_INITF          0
-#define SKIP_ARY_INIT       0
-#define SKIP_SELF_RESTORE   0
-*/
+
 /// handcoded addresses TO BE REMOVED
 
 #define THREAD_INFO            0x20010A9B
@@ -139,6 +128,13 @@ enum CME_STOP_FLAGS
 };
 
 
+enum CME_STOP_PIG_TYPES
+{
+    PIG_TYPE2                        = 2,
+    PIG_TYPE3                        = 3
+};
+
+
 /// Stop Score Board Structure
 typedef struct
 {
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_enter_marks.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_enter_marks.h
index 6e411ccd..ec8a950c 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_enter_marks.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_enter_marks.h
@@ -36,31 +36,27 @@ namespace CME_STOP_ENTRY_MARKS
 
 enum CME_SE_MARKS
 {
-    SE_POLL_PCBMUX_GRANT        =   0x0,
-    SE_QUIESCE_CORE_INTF        =   0x8,
-    SE_STOP_CORE_CLKS       =   0x10,
-    SE_STOP_CORE_GRID       =   0x18,
-    SE_STOP2_DONE       =   0x20,
-    SE_IS0_BEGIN        =   0x28,
-    SE_IS0_END      =   0x30,
-    SE_CATCHUP      =   0x38,
-    SE_CORE_VMIN        =   0x40,
-    SE_STOP3_DONE       =   0x48,
-    SE_POWER_OFF_CORE       =   0x50,
-    SE_STOP4_DONE       =   0x68,
-    SE_IS1_BEGIN        =   0xe0,
-    SE_IS1_END      =   0xe8,
-    SE_PURGE_L2     =   0xf0,
-    SE_IS2_BEGIN        =   0xf8,
-    SE_PURGE_L2_ABORT       =   0x100,
-    SE_PURGE_L2_ABORT_DONE      =   0x108,
-    SE_IS2_END      =   0x110,
-    SE_PURGE_L2_DONE        =   0x118,
-    SE_SGPE_HANDOFF     =   0x120,
-    BEGINSCOPE_STOP_ENTRY       =   0x1f08,
-    ENDSCOPE_STOP_ENTRY     =   0x1f10,
-    STOP_EVENT_HANDLER      =   0x1f18,
-    STOP_DOORBELL_HANDLER       =   0x1f20
+    BEGINSCOPE_STOP_ENTRY       =   0x0,
+    ENDSCOPE_STOP_ENTRY     =   0x8,
+    STOP_EVENT_HANDLER      =   0x10,
+    STOP_DOORBELL_HANDLER       =   0x18,
+    SE_QUIESCE_CORE_INTF        =   0x68,
+    SE_STOP_CORE_CLKS       =   0xe0,
+    SE_STOP_CORE_GRID       =   0xe8,
+    SE_STOP2_DONE       =   0xf0,
+    SE_IS0_BEGIN        =   0xf8,
+    SE_IS0_END      =   0x100,
+    SE_CATCHUP      =   0x108,
+    SE_CORE_VMIN        =   0x110,
+    SE_STOP3_DONE       =   0x118,
+    SE_POWER_OFF_CORE       =   0x120,
+    SE_STOP4_DONE       =   0x128,
+    SE_IS1_BEGIN        =   0x130,
+    SE_IS1_END      =   0x138,
+    SE_PURGE_L2     =   0x140,
+    SE_PURGE_L2_ABORT       =   0x148,
+    SE_PURGE_L2_DONE        =   0x150,
+    SE_SGPE_HANDOFF     =   0x168
 };
 
 
@@ -68,7 +64,10 @@ enum CME_SE_MARKS
 
 const std::vector<CME_SE_MARKS> MARKS =
 {
-    SE_POLL_PCBMUX_GRANT,
+    BEGINSCOPE_STOP_ENTRY,
+    ENDSCOPE_STOP_ENTRY,
+    STOP_EVENT_HANDLER,
+    STOP_DOORBELL_HANDLER,
     SE_QUIESCE_CORE_INTF,
     SE_STOP_CORE_CLKS,
     SE_STOP_CORE_GRID,
@@ -83,16 +82,9 @@ const std::vector<CME_SE_MARKS> MARKS =
     SE_IS1_BEGIN,
     SE_IS1_END,
     SE_PURGE_L2,
-    SE_IS2_BEGIN,
     SE_PURGE_L2_ABORT,
-    SE_PURGE_L2_ABORT_DONE,
-    SE_IS2_END,
     SE_PURGE_L2_DONE,
-    SE_SGPE_HANDOFF,
-    BEGINSCOPE_STOP_ENTRY,
-    ENDSCOPE_STOP_ENTRY,
-    STOP_EVENT_HANDLER,
-    STOP_DOORBELL_HANDLER
+    SE_SGPE_HANDOFF
 };
 
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index a82af15c..3bc44a6d 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -92,6 +92,7 @@ p9_cme_stop_entry()
     // Return error if None of both fired
     if (!core)
     {
+        PK_TRACE("Error: no pm_active fired");
         pk_halt();
     }
 
@@ -139,7 +140,6 @@ p9_cme_stop_entry()
         }
 
         G_cme_stop_record.core_running &= ~core;
-        out32(CME_LCL_LMCR_OR, (core << SHIFT32(15)));
 
         PK_TRACE("SE1: Request Stop Levels[%d %d]",
                  G_cme_stop_record.req_level_c0,
@@ -154,6 +154,7 @@ p9_cme_stop_entry()
             (G_cme_stop_record.req_level_c0 <= STOP_LEVEL_1 ||
              G_cme_stop_record.req_level_c1 <= STOP_LEVEL_1)))
         {
+            PK_TRACE("Error: stop 1 requested to hcode");
             pk_halt();
         }
 
@@ -216,12 +217,6 @@ p9_cme_stop_entry()
         PK_TRACE("SE2: target_lv[%d], deeper_lv[%d], deeper_core[%d]",
                  target_level, deeper_level, deeper_core);
 
-        //=============================
-        MARK_TRAP(SE_POLL_PCBMUX_GRANT)
-        //=============================
-
-        PK_TRACE("SE2.f");
-
         // Poll Infinitely for PCB Mux Grant
         // MF: change watchdog timer in pk to ensure forward progress
         while((core & (in32(CME_LCL_SISR) >> SHIFT32(11))) != core);
@@ -237,7 +232,8 @@ p9_cme_stop_entry()
         out32(CME_LCL_SICR_OR,  core << SHIFT32(1));
         out32(CME_LCL_SICR_CLR, core << SHIFT32(1));
 
-        /// @todo Set LMCR bits 14 and/or 15 (override disables)
+        /// Set LMCR bits 12/13, 14/15 (override disables)
+        out32(CME_LCL_LMCR_OR, ((core << SHIFT32(13)) | (core << SHIFT32(15))));
 
         PK_TRACE("SE2.h");
         // Raise Core-L2 + Core-CC Quiesces
@@ -306,6 +302,8 @@ p9_cme_stop_entry()
         // Switch glsmux to refclk to save clock grid power
         CME_PUTSCOM(C_PPM_CGCR, core, 0);
 
+        // Assert skew sense to skewadjust Fence
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(22));
         // Assert Vital Fence
         CME_PUTSCOM(C_CPLT_CTRL1_OR,  core, BIT64(3));
         // Assert Regional Fences
@@ -720,10 +718,6 @@ p9_cme_stop_entry()
             // Poll for Purged Done
             PK_TRACE("SE5.1b");
 
-            //===========================
-            MARK_TRAP(SE_IS2_BEGIN)
-            //===========================
-
             do
             {
 #if !SKIP_L2_PURGE_ABORT
@@ -742,10 +736,6 @@ p9_cme_stop_entry()
             }
             while((in32(CME_LCL_EISR) & BITS32(22, 2)) != BITS32(22, 2));
 
-            //===================
-            MARK_TRAP(SE_IS2_END)
-            //===================
-
             // Deassert L2+NCU Purges, their possible aborts, NCU tlbie quiesce
             PK_TRACE("SE5.1c");
             out32(CME_LCL_SICR_CLR, BITS32(18, 6));
@@ -773,10 +763,20 @@ p9_cme_stop_entry()
 
         // Send PCB Interrupt per core
         PK_TRACE("SE5.2b");
-        pig.fields.req_intr_type = 2; //0b010: STOP State Change
 
         if (core & CME_MASK_C0)
         {
+            if (G_cme_stop_record.req_level_c0 < STOP_LEVEL_11)
+            {
+                CME_PUTSCOM(CPPM_CPMMR_OR, CME_MASK_C0, BIT64(10));
+                pig.fields.req_intr_type = PIG_TYPE3;
+            }
+            else
+            {
+                CME_PUTSCOM(CPPM_CPMMR_CLR, CME_MASK_C0, BIT64(10));
+                pig.fields.req_intr_type = PIG_TYPE2;
+            }
+
             pig.fields.req_intr_payload = G_cme_stop_record.req_level_c0;
             CME_PUTSCOM(PPM_PIG, CME_MASK_C0, pig.value);
             G_cme_stop_record.core_stopgpe |= core;
@@ -785,6 +785,17 @@ p9_cme_stop_entry()
 
         if (core & CME_MASK_C1)
         {
+            if (G_cme_stop_record.req_level_c1 < STOP_LEVEL_11)
+            {
+                CME_PUTSCOM(CPPM_CPMMR_OR, CME_MASK_C1, BIT64(10));
+                pig.fields.req_intr_type = PIG_TYPE3;
+            }
+            else
+            {
+                CME_PUTSCOM(CPPM_CPMMR_CLR, CME_MASK_C1, BIT64(10));
+                pig.fields.req_intr_type = PIG_TYPE2;
+            }
+
             pig.fields.req_intr_payload = G_cme_stop_record.req_level_c1;
             CME_PUTSCOM(PPM_PIG, CME_MASK_C1, pig.value);
             G_cme_stop_record.core_stopgpe |= core;
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
index dcde15aa..c38399a7 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
@@ -56,6 +56,7 @@ p9_cme_stop_exit()
     // extract wakeup signals, clear status, and mask wakeup interrupts
     wakeup = (in32(CME_LCL_EISR) >> SHIFT32(17)) & 0x3F;
     out32(CME_LCL_EISR_CLR, (wakeup << SHIFT32(17)) | BITS32(24, 2));
+    out32_sh(CME_LCL_EISR_CLR, (in32_sh(CME_LCL_EISR) & BITS32(8, 2)));
 
     // build the core select for wakeup
     pcwu   = (wakeup >> 4) & CME_MASK_BC;
@@ -72,6 +73,7 @@ p9_cme_stop_exit()
     // Code Error: function should never be entered without wakeup source active
     if (!core)
     {
+        PK_TRACE("Error: no wakeup fired");
         pk_halt();
     }
 
@@ -146,16 +148,12 @@ p9_cme_stop_exit()
     // STOP LEVEL 4
     //--------------------------------------------------------------------------
 
-    //======================
-    MARK_TAG(SX_STOP3, core)
-    //======================
-
-    if (deeper_level >= 4)
+    if (deeper_level >= STOP_LEVEL_4)
     {
         PK_TRACE("STOP Level 4+ Sequence");
 
         // if deeper_core is set, then core must be 0b11
-        if (deeper_core && target_level < 4)
+        if (deeper_core && target_level < STOP_LEVEL_4)
         {
             d2u4_flag = 1;
             core      = deeper_core;
@@ -164,10 +162,11 @@ p9_cme_stop_exit()
         do   //catchup loop
         {
 
+#if !SKIP_BCE
             PK_TRACE("BCE Runtime Kickoff");
-
             //right now a blocking call. Need to confirm this.
             instance_scan_init();
+#endif
 
             // todo for catch up case
             //PK_TRACE("X1: Request PCB Arbiter");
@@ -331,7 +330,19 @@ p9_cme_stop_exit()
         }
     }
 
-    // todo STOP LEVEL 3
+    //--------------------------------------------------------------------------
+    // STOP LEVEL 3
+    //--------------------------------------------------------------------------
+    if (deeper_level == STOP_LEVEL_3 || target_level == STOP_LEVEL_3)
+    {
+        //======================
+        MARK_TAG(SX_STOP3, core)
+        //======================
+
+        PK_TRACE("STOP Level 3 Sequence");
+        //Return to full voltage
+        //disable ivrm?
+    }
 
     //--------------------------------------------------------------------------
     // STOP LEVEL 2
@@ -351,7 +362,7 @@ p9_cme_stop_exit()
     // STOP LEVEL 4
     //--------------------------------------------------------------------------
 
-    if (deeper_level >= 4)
+    if (deeper_level >= STOP_LEVEL_4)
     {
         PK_TRACE("STOP Level 4+ Sequence");
 
@@ -370,14 +381,19 @@ p9_cme_stop_exit()
 
         //==========================
         MARK_TAG(SX_BCE_CHECK, core)
-
         //==========================
+
+#if !SKIP_BCE
+        PK_TRACE("BCE Runtime Check");
+
         if( BLOCK_COPY_SUCCESS != isScanRingCopyDone() )
         {
             PK_TRACE("BC2: Copy of Instance Specific Scan ring failed");
             // TODO should return an error code.
         }
 
+#endif
+
         // todo
         //PK_TRACE("X11: XIP Customized Scoms");
         //MARK_TRAP(SX_SCOMCUST)
@@ -406,10 +422,6 @@ p9_cme_stop_exit()
         PK_TRACE("Raise block interrupt to PC");
         out32(CME_LCL_SICR_OR, core << SHIFT32(3));
 
-        //=====================
-        MARK_TRAP(SX_RAM_HRMOR)
-        //=====================
-
         PK_TRACE("RAM HRMOR");
 
         PK_TRACE("Now Wakeup the Core(pm_exit=1)");
@@ -420,9 +432,8 @@ p9_cme_stop_exit()
         while((in32(CME_LCL_EINR)) & (core << SHIFT32(21)));
 
         PK_TRACE("S-Reset all threads");
-        CME_PUTSCOM(DIRECT_CONTROLS, core, BIT64(4));
-        //            BIT64(3)|BIT64(11)|BIT64(19)|BIT64(27));
-        //            BIT64(4)|BIT64(12)|BIT64(20)|BIT64(28));
+        CME_PUTSCOM(DIRECT_CONTROLS, core,
+                    BIT64(4) | BIT64(12) | BIT64(20) | BIT64(28));
 
         //==========================
         MARK_TRAP(SX_SRESET_THREADS)
@@ -435,10 +446,6 @@ p9_cme_stop_exit()
 
         while((~(in32(CME_LCL_EINR))) & (core << SHIFT32(21)));
 
-        //==========================
-        MARK_TRAP(SX_STOP15_THREADS)
-        //==========================
-
         //PK_TRACE("Restore PSSCR back to actual level");
         //PLS here:
         if (core & CME_MASK_C0)
@@ -455,16 +462,16 @@ p9_cme_stop_exit()
         out32(CME_LCL_SICR_CLR, core << SHIFT32(3));
 #endif
 
-        //=========================
-        MARK_TRAP(SX_ENABLE_ANALOG)
-        //=========================
-
         if (d2u4_flag)
         {
             core = CME_MASK_BC;
         }
     }
 
+    //=========================
+    MARK_TRAP(SX_ENABLE_ANALOG)
+    //=========================
+
     //--------------------------------------------------------------------------
     // END OF STOP EXIT
     //--------------------------------------------------------------------------
@@ -472,7 +479,7 @@ p9_cme_stop_exit()
     PK_TRACE("XF: Now Wakeup the Core(pm_exit=1)");
     out32(CME_LCL_SICR_OR, core << SHIFT32(5));
 
-//    PK_TRACE("XF: Polling for Core Wakeup(pm_active=0)");
+    PK_TRACE("XF: Polling for Core Waking up(pm_active=0)");
 
     while((in32(CME_LCL_EINR)) & (core << SHIFT32(21)));
 
@@ -492,7 +499,7 @@ p9_cme_stop_exit()
 
     G_cme_stop_record.core_running |=  core;
     G_cme_stop_record.core_stopgpe &= ~core;
-    out32(CME_LCL_LMCR_CLR, (core << SHIFT32(15)));
+    out32(CME_LCL_LMCR_CLR, ((core << SHIFT32(13)) | (core << SHIFT32(15))));
 
     if (core & CME_MASK_C0)
     {
@@ -506,12 +513,8 @@ p9_cme_stop_exit()
         G_cme_stop_record.act_level_c1 = 0;
     }
 
-    // If not special wakeup, allow core to go back into STOP in the future
-    if (!spwu)
-    {
-        PK_TRACE("XF: Drop pm_exit if special wakeup is not present");
-        out32(CME_LCL_SICR_CLR, core << SHIFT32(5));
-    }
+    PK_TRACE("XF: Drop pm_exit to allow core to run if spwu is not present");
+    out32(CME_LCL_SICR_CLR, core << SHIFT32(5));
 
     //===========================
     MARK_TRAP(ENDSCOPE_STOP_EXIT)
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit_marks.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit_marks.h
index 30f2121d..19ebdb92 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit_marks.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit_marks.h
@@ -36,33 +36,26 @@ namespace CME_STOP_EXIT_MARKS
 
 enum CME_SX_MARKS
 {
-    SX_STOP3        =   0x0,
-    SX_POWERON      =   0x8,
-    SX_POWERON_DONE     =   0x10,
-    SX_CHIPLET_RESET        =   0x18,
-    SX_CHIPLET_RESET_GLSMUX_RESET       =   0x20,
-    SX_CHIPLET_RESET_SCAN0      =   0x28,
-    SX_CATCHUP_A        =   0x30,
-    SX_CHIPLET_INITS        =   0x38,
-    SX_CHIPLET_INIT_SCAN0       =   0x40,
-    SX_CATCHUP_B        =   0x48,
-    SX_ARRAY_INIT       =   0x50,
-    SX_ARRAY_INIT_SUBMODULE     =   0x68,
-    SX_ARRAY_INIT_SCAN0     =   0xe0,
-    SX_FUNC_INIT        =   0xe8,
-    SX_STARTCLOCKS      =   0xf0,
-    SX_STARTCLOCKS_GRID     =   0xf8,
-    SX_STARTCLOCKS_DONE     =   0x100,
-    SX_SCOM_INITS       =   0x108,
-    SX_BCE_CHECK        =   0x110,
-    SX_RUNTIME_INITS        =   0x118,
-    SX_SELF_RESTORE     =   0x120,
-    SX_RAM_HRMOR        =   0x128,
-    SX_SRESET_THREADS       =   0x130,
-    SX_STOP15_THREADS       =   0x138,
-    SX_ENABLE_ANALOG        =   0x140,
-    BEGINSCOPE_STOP_EXIT        =   0x1f28,
-    ENDSCOPE_STOP_EXIT      =   0x1f30
+    BEGINSCOPE_STOP_EXIT        =   0x20,
+    ENDSCOPE_STOP_EXIT      =   0x28,
+    SX_STOP3        =   0x68,
+    SX_POWERON      =   0xe0,
+    SX_CHIPLET_RESET        =   0xe8,
+    SX_CHIPLET_RESET_SCAN0      =   0xf0,
+    SX_CATCHUP_A        =   0xf8,
+    SX_CHIPLET_INITS        =   0x100,
+    SX_CATCHUP_B        =   0x108,
+    SX_ARRAY_INIT       =   0x110,
+    SX_FUNC_INIT        =   0x118,
+    SX_STARTCLOCKS      =   0x120,
+    SX_STARTCLOCKS_ALIGN        =   0x128,
+    SX_STARTCLOCKS_REGION       =   0x130,
+    SX_SCOM_INITS       =   0x138,
+    SX_BCE_CHECK        =   0x140,
+    SX_RUNTIME_INITS        =   0x148,
+    SX_SELF_RESTORE     =   0x150,
+    SX_SRESET_THREADS       =   0x168,
+    SX_ENABLE_ANALOG        =   0x1e0
 };
 
 
@@ -70,33 +63,26 @@ enum CME_SX_MARKS
 
 const std::vector<CME_SX_MARKS> MARKS =
 {
+    BEGINSCOPE_STOP_EXIT,
+    ENDSCOPE_STOP_EXIT,
     SX_STOP3,
     SX_POWERON,
-    SX_POWERON_DONE,
     SX_CHIPLET_RESET,
-    SX_CHIPLET_RESET_GLSMUX_RESET,
     SX_CHIPLET_RESET_SCAN0,
     SX_CATCHUP_A,
     SX_CHIPLET_INITS,
-    SX_CHIPLET_INIT_SCAN0,
     SX_CATCHUP_B,
     SX_ARRAY_INIT,
-    SX_ARRAY_INIT_SUBMODULE,
-    SX_ARRAY_INIT_SCAN0,
     SX_FUNC_INIT,
     SX_STARTCLOCKS,
-    SX_STARTCLOCKS_GRID,
-    SX_STARTCLOCKS_DONE,
+    SX_STARTCLOCKS_ALIGN,
+    SX_STARTCLOCKS_REGION,
     SX_SCOM_INITS,
     SX_BCE_CHECK,
     SX_RUNTIME_INITS,
     SX_SELF_RESTORE,
-    SX_RAM_HRMOR,
     SX_SRESET_THREADS,
-    SX_STOP15_THREADS,
-    SX_ENABLE_ANALOG,
-    BEGINSCOPE_STOP_EXIT,
-    ENDSCOPE_STOP_EXIT
+    SX_ENABLE_ANALOG
 };
 
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
index 59b95954..64bd14cd 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
@@ -29,57 +29,53 @@
 
 extern CmeStopRecord G_cme_stop_record;
 
+// Important: g_eimr_override at any time should mask wakeup interrupts of
+//            running core(s), the override vector should change after each
+//            entry and exit as core state is changed.
+// For Entry, mask the following interrupts via unified interrupt handler:
+//   lower priority interrupts than pm_active, and both pm_active (catchup)
+//   wakeup interrupts of the entering core(s) should still be masked
+//   via g_eimr_override (abortion), stopped core can still exit any time
+//   as their wakeup interrupts should be unmasked
+// After Entry, unmask the following interrupts via pk_irq_vec_restore:
+//   priority group on stack, likely at least both pm_active unmasked
+//   (stopped core cannot get extra pm_active, untouched core can enter)
+//   here needs to use g_eimr_override to mask wakeup of running core(s)
+//   wakeup of the stopped core(s) should be already unmasked by default
+//   (when restored, previous masked wakeups are being unmasked as well)
+// For Exit, mask the following interrupts via unified interrupt handler:
+//   lower priority interrupts than wakeup, including DB2+pm_active(catchup)
+// After Exit, unmask the following interrupts via pk_irq_vec_restore:
+//   priority group on stack, likely at least wakeup and DB2 unmasked
+//   here needs to use g_eimr_override to mask wakeup of exited core(s)
 void
 p9_cme_stop_event_handler(void* arg, PkIrqId irq)
 {
     MARK_TRAP(STOP_EVENT_HANDLER)
-    PK_TRACE("SE-IRQ: %d", irq);
+    PK_TRACE("STOP-IRQ: %d", irq);
     pk_semaphore_post((PkSemaphore*)arg);
-
-    // Important: g_eimr_override at any time should mask wakeup interrupts of
-    //            running core(s), the override vector should change after each
-    //            entry and exit as core state is changed.
-    // For Entry, mask the following interrupts via unified interrupt handler:
-    //   lower priority interrupts than pm_active, and both pm_active (catchup)
-    //   wakeup interrupts of the entering core(s) should still be masked
-    //   via g_eimr_override (abortion), stopped core can still exit any time
-    //   as their wakeup interrupts should be unmasked
-    // After Entry, unmask the following interrupts via pk_irq_vec_restore:
-    //   priority group on stack, likely at least both pm_active unmasked
-    //   (stopped core cannot get extra pm_active, untouched core can enter)
-    //   here needs to use g_eimr_override to mask wakeup of running core(s)
-    //   wakeup of the stopped core(s) should be already unmasked by default
-    //   (when restored, previous masked wakeups are being unmasked as well)
-    // For Exit, mask the following interrupts via unified interrupt handler:
-    //   lower priority interrupts than wakeup, including DB2+pm_active(catchup)
-    // After Exit, unmask the following interrupts via pk_irq_vec_restore:
-    //   priority group on stack, likely at least wakeup and DB2 unmasked
-    //   here needs to use g_eimr_override to mask wakeup of exited core(s)
 }
 
 void
 p9_cme_stop_doorbell_handler(void* arg, PkIrqId irq)
 {
-    int               rc = 0;
+    uint32_t          db1;
     PkMachineContext  ctx;
+
     MARK_TRAP(STOP_DOORBELL_HANDLER)
     PK_TRACE("DB-IRQ: %d", irq);
 
-    out32_sh(CME_LCL_EIMR_OR, BIT32(irq - 32));
-    out32_sh(CME_LCL_EISR_CLR, BIT32(irq - 32));
+    db1 = in32_sh(CME_LCL_EISR);
+    g_eimr_override |= BITS64(40, 2);
 
-    if (irq == IRQ_DB1_C0)
+    if (db1 & BIT32(8))
     {
-        CME_PUTSCOM(CPPM_CMEDB1, CME_MASK_C0, 0);
         g_eimr_override &= ~IRQ_VEC_WAKE_C0;
-        //out32(CME_LCL_EIMR_CLR, BIT32(12) | BIT32(14) | BIT32(16));
     }
 
-    if (irq == IRQ_DB1_C1)
+    if (db1 & BIT32(9))
     {
-        CME_PUTSCOM(CPPM_CMEDB1, CME_MASK_C1, 0);
         g_eimr_override &= ~IRQ_VEC_WAKE_C1;
-        //out32(CME_LCL_EIMR_CLR, BIT32(13) | BIT32(15) | BIT32(17));
     }
 
     pk_irq_vec_restore(&ctx);
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_arrayinit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_arrayinit.c
index e10997fa..b64f30b9 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_arrayinit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_arrayinit.c
@@ -30,68 +30,63 @@ int
 p9_hcd_core_arrayinit(uint32_t core)
 {
     int rc = CME_STOP_SUCCESS;
-#if !EPM_P9_TUNING
-    uint64_t scom_data;
-
-    PK_TRACE("Setup ABISTMUX_SEL");
-    CME_PUTSCOM(PERV_CPLT_CTRL0_OR, core, BIT64(0));
-
-    PK_TRACE("setup ABIST modes");
-    CME_GETSCOM(PERV_BIST, core, CME_SCOM_OR, scom_data);
-    scom_data &= ~BIT64(0) ;
-    scom_data |= BITS64(1, 2);
-    scom_data |= BITS64(6, 2);
-    CME_PUTSCOM(PERV_BIST, core, scom_data);
-
-    PK_TRACE("Setup all Clock Domains and Clock Types");
-    CME_GETSCOM(PERV_CLK_REGION, core, CME_SCOM_OR, scom_data);
-    scom_data |= BITS64(6, 2);
-    scom_data |= BITS64(48, 3);
-    CME_PUTSCOM(PERV_CLK_REGION, core, scom_data);
-
-    PK_TRACE("Setup: loopcount , OPCG engine start ABIST, run-N mode");
-    CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
-    scom_data = 0x8002000000042FFF;
-    CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
-
-    PK_TRACE("Setup IDLE count");
-    CME_GETSCOM(PERV_OPCG_REG1, core, CME_SCOM_OR, scom_data);
-    scom_data &= 0x000000000FFFFFFF;
-    scom_data |= 0x0000000F00000000;
-    CME_PUTSCOM(PERV_OPCG_REG1, core, scom_data);
-
-    PK_TRACE("opcg go");
-    CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
-    scom_data |= BIT64(1);
-    CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
-
-    PK_TRACE("Poll OPCG done bit to check for run-N completeness");
-
-    do
-    {
-        CME_GETSCOM(PERV_CPLT_STAT0, core, CME_SCOM_AND, scom_data);
-    }
-    while(!(scom_data & BIT64(8)));
-
-    MARK_TRAP(SX_ARRAY_INIT_SUBMODULE)
-
-    PK_TRACE("OPCG done, clear Run-N mode");
-    CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
-    scom_data &= ~(BIT64(0) | BIT64(14) | BITS64(21, 43));
-    CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
-
-    PK_TRACE("clear all clock REGIONS and type");
-    CME_PUTSCOM(PERV_CLK_REGION, core, 0);
-
-    PK_TRACE("clear ABISTCLK_MUXSEL");
-    CME_PUTSCOM(PERV_CPLT_CTRL0_CLEAR, core, BIT64(0));
-
-    PK_TRACE("clear BIST REGISTER");
-    CME_PUTSCOM(PERV_BIST, core, 0);
-#endif
-#if !SKIP_SCAN0
-    MARK_TRAP(SX_ARRAY_INIT_SCAN0)
-#endif
+    /* @todo minic arrayinit module
+        uint64_t scom_data;
+
+        PK_TRACE("Setup ABISTMUX_SEL");
+        CME_PUTSCOM(PERV_CPLT_CTRL0_OR, core, BIT64(0));
+
+        PK_TRACE("setup ABIST modes");
+        CME_GETSCOM(PERV_BIST, core, CME_SCOM_OR, scom_data);
+        scom_data &= ~BIT64(0) ;
+        scom_data |= BITS64(1, 2);
+        scom_data |= BITS64(6, 2);
+        CME_PUTSCOM(PERV_BIST, core, scom_data);
+
+        PK_TRACE("Setup all Clock Domains and Clock Types");
+        CME_GETSCOM(PERV_CLK_REGION, core, CME_SCOM_OR, scom_data);
+        scom_data |= BITS64(6, 2);
+        scom_data |= BITS64(48, 3);
+        CME_PUTSCOM(PERV_CLK_REGION, core, scom_data);
+
+        PK_TRACE("Setup: loopcount , OPCG engine start ABIST, run-N mode");
+        CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
+        scom_data = 0x8002000000042FFF;
+        CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
+
+        PK_TRACE("Setup IDLE count");
+        CME_GETSCOM(PERV_OPCG_REG1, core, CME_SCOM_OR, scom_data);
+        scom_data &= 0x000000000FFFFFFF;
+        scom_data |= 0x0000000F00000000;
+        CME_PUTSCOM(PERV_OPCG_REG1, core, scom_data);
+
+        PK_TRACE("opcg go");
+        CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
+        scom_data |= BIT64(1);
+        CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
+
+        PK_TRACE("Poll OPCG done bit to check for run-N completeness");
+
+        do
+        {
+            CME_GETSCOM(PERV_CPLT_STAT0, core, CME_SCOM_AND, scom_data);
+        }
+        while(!(scom_data & BIT64(8)));
+
+        PK_TRACE("OPCG done, clear Run-N mode");
+        CME_GETSCOM(PERV_OPCG_REG0, core, CME_SCOM_OR, scom_data);
+        scom_data &= ~(BIT64(0) | BIT64(14) | BITS64(21, 43));
+        CME_PUTSCOM(PERV_OPCG_REG0, core, scom_data);
+
+        PK_TRACE("clear all clock REGIONS and type");
+        CME_PUTSCOM(PERV_CLK_REGION, core, 0);
+
+        PK_TRACE("clear ABISTCLK_MUXSEL");
+        CME_PUTSCOM(PERV_CPLT_CTRL0_CLEAR, core, BIT64(0));
+
+        PK_TRACE("clear BIST REGISTER");
+        CME_PUTSCOM(PERV_BIST, core, 0);
+    */
 
     return rc;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_init.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_init.c
index 152cbb2b..3d1ef150 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_init.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_init.c
@@ -31,10 +31,5 @@ p9_hcd_core_chiplet_init(uint32_t core)
 {
     int rc = CME_STOP_SUCCESS;
 
-#if !SKIP_SCAN0
-    // Marker for scan0
-    MARK_TRAP(SX_CHIPLET_INIT_SCAN0)
-#endif
-
     return rc;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
index 50c0e058..a97ac8da 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_chiplet_reset.c
@@ -33,8 +33,9 @@ p9_hcd_core_chiplet_reset(uint32_t core)
     //uint64_t scom_data,
     uint32_t loop;
 
-    PK_TRACE("Init NET_CTRL0[1-5,12-14,18,22,26],step needed for hotplug");
-    CME_PUTSCOM(CPPM_NC0INDIR_OR, core, NET_CTRL0_INIT_VECTOR);
+    PK_TRACE("Init NET_CTRL0[1,3-5,11-14,18,22,26],step needed for hotplug");
+    CME_PUTSCOM(CPPM_NC0INDIR_OR,  core, C_NET_CTRL0_INIT_VECTOR);
+    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, ~(C_NET_CTRL0_INIT_VECTOR | BIT64(2)));
 
     PK_TRACE("Flip core glsmux to refclk via PPM_CGCR[3]");
     CME_PUTSCOM(C_PPM_CGCR, core, BIT64(0));
@@ -42,17 +43,8 @@ p9_hcd_core_chiplet_reset(uint32_t core)
     PK_TRACE("Assert core progdly and DCC bypass via NET_CTRL1[1,2]");
     CME_PUTSCOM(CPPM_NC1INDIR_OR, core, BITS64(1, 2));
 
-    PK_TRACE("Assert core DCC reset via NET_CTRL0[2]");
-    CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(2));
-
     PK_TRACE("Drop vital thold via NET_CTRL0[16]");
     CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(16));
-    /*
-        PK_TRACE("ONLY till TP030: SET VITL_PHASE=1");
-        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(8));
-        PPE_WAIT_CORE_CYCLES(loop, 50);
-    */
-    MARK_TRAP(SX_CHIPLET_RESET_GLSMUX_RESET)
 
     PK_TRACE("Drop core glsmux reset via PPM_CGCR[0]");
     CME_PUTSCOM(C_PPM_CGCR, core, 0);
@@ -72,13 +64,7 @@ p9_hcd_core_chiplet_reset(uint32_t core)
 
     PK_TRACE("Drop PCB fence via NET_CTRL0[25]");
     CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(25));
-    /*
-        PK_TRACE("ONLY till TP030: SET SYNC_PULSE_DELAY=0b0011");
-        CME_GETSCOM(C_SYNC_CONFIG, core, CME_SCOM_AND, scom_data);
-        scom_data = scom_data | 0x3000000000000000;
-        scom_data = scom_data & 0x3FFFFFFFFFFFFFFF;
-        CME_PUTSCOM(C_SYNC_CONFIG, core, scom_data);
-    */
+
 #if !SKIP_SCAN0
     // Marker for scan0
     MARK_TRAP(SX_CHIPLET_RESET_SCAN0)
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
index b24a192e..b2a1acb2 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_poweron.c
@@ -66,8 +66,6 @@ p9_hcd_core_poweron(uint32_t core)
     }
     while(!(scom_data & BIT64(42)));
 
-    MARK_TRAP(SX_POWERON_DONE)
-
     // vdd_pfet_force_state = 00 (Nop)
     PK_TRACE("Turn Off Force Von");
     CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
index 3ab36b7f..06aeea19 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
@@ -48,6 +48,9 @@ p9_hcd_core_startclocks(uint32_t core)
     PK_TRACE("Drop vital fences via CPLT_CTRL1[3]");
     CME_PUTSCOM(C_CPLT_CTRL1_CLEAR, core, BIT64(3));
 
+    PK_TRACE("Drop skew sense to skew adjust fence via NET_CTRL0[22]");
+    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(22));
+
     PK_TRACE("Assert core clock sync enable via CPPM_CACCR[15]");
     CME_PUTSCOM(CPPM_CACCR_OR, core, BIT64(15));
 
@@ -59,6 +62,8 @@ p9_hcd_core_startclocks(uint32_t core)
     }
     while(~scom_data & BIT64(13));
 
+    MARK_TRAP(SX_STARTCLOCKS_ALIGN)
+
     PK_TRACE("Reset abstclk & syncclk muxsel(io_clk_sel) via CPLT_CTRL0[0:1]");
     CME_PUTSCOM(C_CPLT_CTRL0_CLEAR, core, BITS64(0, 2));
 
@@ -85,6 +90,8 @@ p9_hcd_core_startclocks(uint32_t core)
     }
     while(~scom_data & BIT64(9));
 
+    MARK_TRAP(SX_STARTCLOCKS_REGION)
+
     PK_TRACE("Drop force_align via CPLT_CTRL0[3]");
     CME_PUTSCOM(C_CPLT_CTRL0_CLEAR, core, BIT64(3));
     PPE_WAIT_CORE_CYCLES(loop, 450);
@@ -112,8 +119,6 @@ p9_hcd_core_startclocks(uint32_t core)
 
     PK_TRACE("Core clock is now running");
 
-    MARK_TRAP(SX_STARTCLOCKS_DONE)
-
     PK_TRACE("Drop chiplet fence via NC0INDIR[18]");
     CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(18));