STOP: Add Core quiesce workaround to CME Fit Timer

-- p7:  Added a check for LSU and Nest quiesce as well
-- p8:  Added clear of the TSR FIS to IOTA fit handler
-- p9:  Added stop workaround on checkstop or special attention
-- p10: rebase on disable flag support
-- p11:
1) enable for Cumulus 1.0 and Nimbus 2.0 & 2.1 only
2) disable the livelock buster if special wakeup done is asserted
3) clear the fit_counter when special wakeup done
   or stop entry is done on either core
4) delete CME_FLAGS_DISABLE_PERIODIC_CORE_QUIESCE and
   instead look at bit 2 of CPPMR with dualcast OR
   unless nimbus dd2.0 then you have to
   read both cores and OR because of the bug
-- p12:
1) no longer have PGPE sample OCC Flag bit 13 (back out the change)
2) have SGPE init set CPMMR bit 2 in response to the attribute
   instead of CME flag bit 23.
3) add cme_flags[12] as workaround active bit (nor of both cpmmr[2])
-- p13:
   kills the CME flag, OCC flag and SGPE header flag
   diff commit will establish attr-> cpmmr[2] simple flow
   cme only read cpmmr[2]
-- p14:
   in case of stop0/1, assert pm_exit and poll for it to wake core up
-- p15: rebase
-- p16: add drop pm_exit after workaround
-- p17: add pm_exit delay, remove pm_active=1 poll
        remove some PK_TRACEs, change to unified compiler flag

Change-Id: I855f7f8e4b5ce7275511d581b8e68d10217ab601
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/44965
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Juan R. Medina <jrmedina@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

12 files changed, 435 insertions, 106 deletions

diff --git a/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h b/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
index 9e6ffd2f..3d3ed472 100644
--- a/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
+++ b/import/chips/p9/procedures/hwp/lib/p9_pm_hcd_flags.h
@@ -143,31 +143,6 @@ namespace p9hcd
 {
 #endif
 
-//Enum form of CME FLAGs.
-enum PM_CME_CMEFLG_DEFS
-{
-    CME_STOP_READY                      = 0,
-    CME_PMCR_READY                      = 1,
-    CME_QMGR_READY                      = 2,
-    CME_QMGR_MASTER                     = 3,
-    CME_RCLK_OPERABLE                   = 4,
-    CME_IVRM_OPERABLE                   = 5,
-    CME_VDM_OPERABLE                    = 6,
-    CME_OCC_HB_SAFE_MODE                = 7,
-    CME_BLOCK_STOP_EXIT_ENABLED_C0      = 8,
-    CME_BLOCK_STOP_EXIT_ENABLED_C1      = 9,
-    CME_BLOCK_STOP_ENTRY_ENABLED_C0     = 10,
-    CME_BLOCK_STOP_ENTRY_ENABLED_C1     = 11,
-    // Reserve 12:24
-    CME_FREQ_UPDATE_DISABLE             = 25,
-    CME_EX_ID                           = 26,
-    CME_SIBLING_FUNCTIONAL              = 27,
-    CME_STOP_ENTRY_FIRST_C0             = 28,
-    CME_STOP_ENTRY_FIRST_C1             = 29,
-    CME_CORE0_GOOD                      = 30,
-    CME_CORE1_GOOD                      = 31
-};
-
 //Enum form of OCC FLAGs.
 enum PM_GPE_OCCFLG_DEFS
 {
@@ -181,7 +156,6 @@ enum PM_GPE_OCCFLG_DEFS
     SGPE_IGNORE_STOP_ACTION             = 10,
     SGPE_IGNORE_STOP_EXITS              = 11,
     SGPE_IGNORE_STOP_ENTRIES            = 12,
-    OCCFLG_CORE_QUIESCE_WORKARND_DIS    = 13,
     SGPE_24_7_ACTIVATE                  = 14,
     SGPE_24_7_ACTIVE                    = 15,
     PIB_I2C_MASTER_ENGINE_1_LOCK_BIT0   = 16, //BIT0 ored BIT1 gives the field
@@ -223,9 +197,11 @@ enum PM_CME_FLAGS_DEFS
     CME_FLAGS_IVRM_OPERABLE                 = 5,
     CME_FLAGS_VDM_OPERABLE                  = 6,
     CME_FLAGS_OCC_HB_SAFE_MODE              = 7,
-    CME_FLAGS_BLOCK_WKUP_C0                 = 8,
-    CME_FLAGS_BLOCK_WKUP_C1                 = 9,
-    CME_CORE_QUIESCE_WORKARND_DIS           = 23,
+    CME_FLAGS_STOP_BLOCK_EXIT_C0            = 8,
+    CME_FLAGS_STOP_BLOCK_EXIT_C1            = 9,
+    CME_FLAGS_STOP_BLOCK_ENTRY_C0           = 10,
+    CME_FLAGS_STOP_BLOCK_ENTRY_C1           = 11,
+    CME_FLAGS_CORE_QUIESCE_ACTIVE           = 12,
     CME_FLAGS_PSTATES_ENABLED               = 24,
     CME_FLAGS_FREQ_UPDT_DISABLE             = 25,
     CME_FLAGS_EX_ID                         = 26,
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme.h b/import/chips/p9/procedures/ppe_closed/cme/p9_cme.h
index 8278fbdc..33172533 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme.h
@@ -27,10 +27,33 @@
 
 #include "pk.h"
 #include "p9_pm_hcd_flags.h"
+
+#ifdef PCQW_ENABLE
+
+typedef struct
+{
+    uint32_t core_quiesce_fit_trigger;
+    uint32_t core_quiesce_total_count;
+    uint32_t core_quiesce_time_latest;
+    uint32_t core_quiesce_time_max;
+    uint32_t core_quiesce_time_min;
+} CmeFitRecord;
+
+#endif
+
+
+
 typedef struct
 {
     // target mask of enabled cores, used to filter 2bit core select in scom address
-    uint32_t      core_enabled;
+    uint32_t     core_enabled;
+
+#ifdef PCQW_ENABLE
+
+    CmeFitRecord fit_record;
+
+#endif
+
 } CmeRecord;
 
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
index 59a837f1..617e5753 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
@@ -30,7 +30,16 @@
 
 // CME Pstate Header and Structure
 #include "p9_cme.h"
-CmeRecord G_cme_record ;
+
+#ifdef PCQW_ENABLE
+
+CmeRecord G_cme_record = {0, {0, 0, 0, 0, 0xFFFFFFFF}};
+
+#else
+
+CmeRecord G_cme_record = {0};
+
+#endif
 
 // CME Pstate Header and Structure
 #include "p9_cme_pstate.h"
@@ -40,11 +49,336 @@ CmePstateRecord G_cme_pstate_record;
 #include "p9_cme_stop.h"
 CmeStopRecord G_cme_stop_record __attribute__((section (".dump_ptrs"))) = {{0}, {0}, 0, 0, 0, 0, 0, 0, 0, {0}};
 
+// NDD2: OOB bits wired to SISR
+//       not implemented in DD1
+// bit0 is System checkstop
+// bit1 is Recoverable Error
+// bit2 is Special Attention
+// bit3 is Core Checkstop
+#define bad_error_present (((in32   (CME_LCL_SISR) & (  BIT32(12) |   BITS32(14,2)))) || \
+                           ((in32_sh(CME_LCL_SISR) & (BIT64SH(60) | BITS64SH(62,2)))))
+
+
+#if !DISABLE_PERIODIC_CORE_QUIESCE && (NIMBUS_DD_LEVEL == 20 || NIMBUS_DD_LEVEL == 21 || CUMULUS_DD_LEVEL == 10)
+
+inline static
+void periodic_core_quiesce_workaround()
+{
+    uint32_t core;
+    uint32_t core_accessible;
+    uint32_t fused_core_mode;
+    uint32_t spattn_offset;
+    uint32_t spattn[2];
+    uint32_t maint_mode[2];
+    uint32_t time_stamp[2];
+    data64_t scom_data;
+    uint32_t sample_error = 0;
+
+    PK_TRACE_INF("FIT: Periodic Core Quiesce Workaround");
+
+    CME_GETSCOM_AND(CPPM_CPMMR, CME_MASK_BC, scom_data.value);
+    fused_core_mode = scom_data.words.upper & BIT32(9);
+
+    PK_TRACE_DBG("PCQW: Fused Core Mode[%x]", fused_core_mode);
+
+    //0) in case in stop0/1 that we dont know about
+
+    PK_TRACE("PCQW: Assert block interrupt to PC via SICR[2/3]");
+    out32(CME_LCL_SICR_OR, CME_MASK_BC << SHIFT32(3));
+
+    PK_TRACE("PCQW: Waking up the core(pm_exit=1) via SICR[4/5]");
+    out32(CME_LCL_SICR_OR, CME_MASK_BC << SHIFT32(5));
+
+    CME_PM_EXIT_DELAY
+
+    PK_TRACE("PCQW: Polling for core wakeup(pm_active=0) via EINR[20/21]");
+
+    while((in32(CME_LCL_EINR)) & (CME_MASK_BC << SHIFT32(21)));
+
+    //1) Acquire Pcb Mux
+
+    core_accessible = ((~in32(CME_LCL_SISR)) >> SHIFT32(11)) & CME_MASK_BC;
+
+    PK_TRACE("PCQW: Request PCB Mux via SICR[10/11]");
+    out32(CME_LCL_SICR_OR, core_accessible << SHIFT32(11));
+
+    // Poll Infinitely for PCB Mux Grant
+    while((core_accessible & (in32(CME_LCL_SISR) >> SHIFT32(11))) != core_accessible);
+
+    PK_TRACE("PCQW: PCB Mux Granted");
+
+    //2) Read RAS_STATUS Scom Addr(20:31) = x0A02
+    //   bit (0 + 8*T) where (T= thread) CORE_MAINT_MODE
+    //   to find out which threads are in maintenance mode
+
+    // vector = 0 + 8*T as a shifting base used below
+#define THREAD_VECTOR (BIT32(0)|BIT32(8)|BIT32(16)|BIT32(24))
+
+    for(core = CME_MASK_C0; core > 0; core--)
+    {
+        CME_GETSCOM(RAS_STATUS, core, scom_data.value) ;
+        maint_mode[core & 1] = scom_data.words.upper & THREAD_VECTOR;
+    }
+
+
+    //3) Write DIRECT_CONTROLS Scom Addr(20:31) = x0A9C
+    //   bit (7 + 8*T) where (T= thread) DC_CORE_STOP for ALL threads.
+    //   This will quiesce the active threads,
+    //   put all threads into core maintenance mode,
+    //   and eventually quiesce the entire core.
+
+    scom_data.words.lower = 0;
+    scom_data.words.upper = THREAD_VECTOR >> 7;
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    for(core = CME_MASK_C0; core > 0; core--)
+    {
+
+#else
+
+    core = CME_MASK_BC;
+
+#endif
+
+        CME_PUTSCOM_NOP(DIRECT_CONTROLS, core, scom_data.value);
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    }
+
+#endif
+
+
+    //4) Loop on RAS_STATUS Scom Addr(20:31) = x0A02
+    //   until bit(1 + 8*T) THREAD_QUIESCE are all active b1
+
+    time_stamp[0] = in32(CME_LCL_TBR);
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    for(core = CME_MASK_C0; core > 0; core--)
+    {
+
+#else
+
+    core = CME_MASK_BC;
+
+#endif
+
+#define THREAD_VECTOR_CHECK (THREAD_VECTOR>>1 | THREAD_VECTOR>>3)
+
+
+        do
+        {
+            CME_GETSCOM_AND(RAS_STATUS, core, scom_data.value);
+        }
+
+        while((((scom_data.words.upper& THREAD_VECTOR_CHECK) != THREAD_VECTOR_CHECK) ||  //THREAD_ and LSU_QUIESCE must be ones
+               ((scom_data.words.lower& BIT64SH(32))))  // NEST_ACTIVE must be zero
+              && !(sample_error = bad_error_present)
+             );
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    }
+
+#endif
+
+    time_stamp[1] = in32(CME_LCL_TBR);
+
+    if (!sample_error)
+    {
+        PK_TRACE_DBG("FIT: Both Cores Quiesced");
+    }
+    else
+    {
+        PK_TRACE_DBG("FIT: Error while trying to Quiesce Cores");
+    }
+
+
+    for(core = CME_MASK_C0; core > 0; core--)
+    {
+
+        //5) Read SPATTN Scom Addr(20:31) = x0A98 to check for ATTN
+        //   (need to do this after all threads quiesce to close windows)
+
+        CME_GETSCOM(SPATTN_READ, core, scom_data.value);
+
+        if (fused_core_mode)
+        {
+            //Fused Core Mode
+            // C0 vtid0=ltid0 bit1  -> tv.bit0
+            // C0 vtid1=ltid2 bit9  -> tv.bit8
+            // C0 vtid2=ltid4 bit17 -> tv.bit16
+            // C0 vtid3=ltid6 bit25 -> tv.bit24
+            //
+            // C1 vtid0=ltid1 bit5  -> tv.bit0
+            // C1 vtid1=ltid3 bit13 -> tv.bit8
+            // C1 vtid2=ltid5 bit21 -> tv.bit16
+            // C1 vtid3=ltid7 bit29 -> tv.bit24
+            spattn_offset = ((core & 1) << 2) + 1; // C0:1, C1:5
+            spattn[core & 1] = ((scom_data.words.upper & BIT32((0 + spattn_offset))) << spattn_offset) | //0
+                               ((scom_data.words.upper & BIT32((8 + spattn_offset))) << spattn_offset) | //8
+                               ((scom_data.words.upper & BIT32((16 + spattn_offset))) << spattn_offset) | //16
+                               ((scom_data.words.upper & BIT32((24 + spattn_offset))) << spattn_offset); //24
+        }
+        else
+        {
+            // Normal Mode
+            // vtid0=ltid0 bit1  -> tv.bit0
+            // vtid1=ltid1 bit5  -> tv.bit8
+            // vtid2=ltid2 bit9  -> tv.bit16
+            // vtid3=ltid3 bit13 -> tv.bit24
+            spattn[core & 1] = ((scom_data.words.upper & BIT32(1))  << 1 ) | //0
+                               ((scom_data.words.upper & BIT32(5))  >> 3 ) | //8
+                               ((scom_data.words.upper & BIT32(9))  >> 7 ) | //16
+                               ((scom_data.words.upper & BIT32(13)) >> 11);  //24
+        }
+
+
+        //6) Write DIRECT_CONTROLS Scom Addr(20:31) = x0A9C
+        //   bit (3 + 8*T) where (T= thread) DC_CLEAR_MAINT for all threads
+        //   which were not in maintenance mode in step 1 AND do not have ATTN set in step 4
+
+        scom_data.words.lower = 0;
+        scom_data.words.upper =
+            (THREAD_VECTOR & (~maint_mode[core & 1]) & (~spattn[core & 1])) >> 3;
+        CME_PUTSCOM_NOP(DIRECT_CONTROLS, core, scom_data.value);
+    }
+
+    PK_TRACE_DBG("FIT: Both Cores Started");
+
+    //7) Drop pm_exit
+
+    PK_TRACE("PCQW: Drop pm_exit via SICR[4/5]");
+    out32(CME_LCL_SICR_CLR, CME_MASK_BC << SHIFT32(5));
+
+    PK_TRACE("PCQW: Drop block interrupt to PC via SICR[2/3]");
+    out32(CME_LCL_SICR_CLR, CME_MASK_BC << SHIFT32(3));
+
+    //8) Release Pcb Mux on Both Cores
+
+    PK_TRACE("PCQW: Release PCB Mux back on Both Cores via SICR[10/11]");
+    out32(CME_LCL_SICR_CLR, core_accessible << SHIFT32(11));
+
+    while((core_accessible & ~(in32(CME_LCL_SISR) >> SHIFT32(11))) != core_accessible);
+
+    PK_TRACE("PCQW: PCB Mux Released on Both Cores");
+
+
+    G_cme_record.fit_record.core_quiesce_total_count += 1;
+
+    //Profile time
+
+    if (time_stamp[1] > time_stamp[0])
+    {
+        G_cme_record.fit_record.core_quiesce_time_latest =
+            time_stamp[1] - time_stamp[0];
+    }
+    else
+    {
+        G_cme_record.fit_record.core_quiesce_time_latest =
+            0xFFFFFFFF - time_stamp[0] + time_stamp[1] + 1;
+    }
+
+    if (G_cme_record.fit_record.core_quiesce_time_latest <
+        G_cme_record.fit_record.core_quiesce_time_min)
+    {
+        G_cme_record.fit_record.core_quiesce_time_min =
+            G_cme_record.fit_record.core_quiesce_time_latest;
+    }
+    else if (G_cme_record.fit_record.core_quiesce_time_latest >
+             G_cme_record.fit_record.core_quiesce_time_max)
+    {
+        G_cme_record.fit_record.core_quiesce_time_max =
+            G_cme_record.fit_record.core_quiesce_time_latest;
+    }
+}
+
+#endif
+
+
 void fit_handler()
 {
-    // Default is to do nothing
+    mtspr(SPRN_TSR, TSR_FIS);
+
+#ifdef PCQW_ENABLE
+
+    uint32_t core_quiesce_cpmmr_disable = 0;
+    uint32_t core;
+    uint32_t scom_op;
+    data64_t scom_data;
+
+    PK_TRACE_DBG("FIT Timer Handler");
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    scom_op = CME_SCOM_NOP;
+
+    for(core = CME_MASK_C0; core > 0; core--)
+    {
+
+#else
+
+    scom_op = CME_SCOM_OR;
+    core    = CME_MASK_BC;
+
+#endif
+
+        CME_GETSCOM_OP(CPPM_CPMMR, core, scom_op, scom_data.value);
+
+        if (scom_data.words.upper & BIT32(2))
+        {
+            core_quiesce_cpmmr_disable = 1;
+        }
+
+#if NIMBUS_DD_LEVEL == 20 || DISABLE_CME_DUAL_CAST == 1
+
+    }
+
+#endif
+
+    if (!core_quiesce_cpmmr_disable)
+    {
+        out32(CME_LCL_FLAGS_OR,  BIT32(CME_FLAGS_CORE_QUIESCE_ACTIVE));
+    }
+    else
+    {
+        out32(CME_LCL_FLAGS_CLR, BIT32(CME_FLAGS_CORE_QUIESCE_ACTIVE));
+    }
+
+    // only run workaround if
+    // 1) both cores are enabled
+    // 2) both cores are running
+    //    (stop entry clears the counter)
+    // 3) both cores doesnt have special_wakeup_done asserted
+    //    (spwu_done clears the counter)
+    // 4) both core doesnt have cpmmr[2] asserted
+    // 5) no bad error occurs
+    if((G_cme_record.core_enabled == CME_MASK_BC) &&
+       (G_cme_stop_record.core_running == CME_MASK_BC) &&
+       (!(in32(CME_LCL_SISR) & BITS32(16, 2))) &&
+       (!core_quiesce_cpmmr_disable) &&
+       (!bad_error_present))
+    {
+        if (G_cme_record.fit_record.core_quiesce_fit_trigger < 10)
+        {
+            G_cme_record.fit_record.core_quiesce_fit_trigger++;
+        }
+        else
+        {
+            G_cme_record.fit_record.core_quiesce_fit_trigger = 0;
+            periodic_core_quiesce_workaround();
+        }
+    }
+
+#endif
+
 }
 
+
+
 void dec_handler()
 {
     // Currently not available
@@ -78,18 +412,29 @@ IOTA_TASK(ext_handler), // bits 0-6   default
 
 int main()
 {
+    // Register Timer Handlers
     IOTA_DEC_HANDLER(dec_handler);
     IOTA_FIT_HANDLER(fit_handler);
 
     PK_TRACE("E>CME MAIN");
 
+    // Clear SPRG0
+    ppe42_app_ctx_set(0);
+
+    G_cme_record.core_enabled = in32(CME_LCL_FLAGS) &
+                                (BIT32(CME_FLAGS_CORE0_GOOD) | BIT32(CME_FLAGS_CORE1_GOOD));
+    PK_TRACE("CME Register Partial Good Cores[%d]", G_cme_record.core_enabled);
+
 #if defined(USE_CME_QUEUED_SCOM) || defined(USE_CME_QUEUED_SCAN)
+    PK_TRACE("CME Enabling Queued Scom/Scan");
     out32(CME_LCL_LMCR_OR, BITS32(8, 2));
 #endif
 
-    //Read which cores are good
-    G_cme_record.core_enabled = in32(CME_LCL_FLAGS) &
-                                (BIT32(CME_FLAGS_CORE0_GOOD) | BIT32(CME_FLAGS_CORE1_GOOD));
+    PK_TRACE("Set Watch Dog Timer Rate to 6 and FIT Timer Rate to 8");
+    out32(CME_LCL_TSEL, (BITS32(1, 2) | BIT32(4)));
+
+    PK_TRACE("Enable DEC/FIT/Watchdog Timer");
+    mtspr(SPRN_TCR, (TCR_DIE | TCR_FIE));
 
     p9_cme_stop_init();
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
index 434af8a9..6b4ade34 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
@@ -36,9 +36,14 @@
 
 // Debug Switches
 
-#define TEST_ONLY_BCE_IRR            0
-#define DISABLE_CME_DUAL_CAST        0
-#define DISABLE_CORE_XSTOP_INJECTION 0
+#define TEST_ONLY_BCE_IRR                0
+#define DISABLE_CME_DUAL_CAST            0
+#define DISABLE_CORE_XSTOP_INJECTION     0
+#define DISABLE_PERIODIC_CORE_QUIESCE    0
+
+#if !DISABLE_PERIODIC_CORE_QUIESCE && (NIMBUS_DD_LEVEL == 20 || NIMBUS_DD_LEVEL == 21 || CUMULUS_DD_LEVEL == 10)
+    #define PCQW_ENABLE
+#endif
 
 // --------------------
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
index 245b38b2..e35ee298 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
@@ -87,6 +87,7 @@
 #define CORE_FIRMASK           0x20010A43
 #define CORE_ACTION0           0x20010A46
 #define CORE_ACTION1           0x20010A47
+#define SPATTN_READ            0x20010A99
 #define SPATTN_MASK            0x20010A9A
 
 #define C_HANG_CONTROL         0x20010A00
@@ -151,6 +152,12 @@
     p9_cme_stop_core_error_handler(core, core_error, panic_code);
 #endif
 
+#define CME_PM_EXIT_DELAY         \
+    asm volatile ("tw 0, 0, 0");  \
+    asm volatile ("tw 0, 0, 0");  \
+    asm volatile ("tw 0, 0, 0");  \
+    asm volatile ("tw 0, 0, 0");
+
 enum CME_IRQ_VECTORS
 {
     // if auto mask eimr.spwu else never mask eimr.spwu
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index 96ef1741..2b7dced3 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -408,6 +408,12 @@ p9_cme_stop_entry()
             // Mark core as to be stopped
             G_cme_stop_record.core_running &= ~core;
 
+#ifdef PCQW_ENABLE
+
+            G_cme_record.fit_record.core_quiesce_fit_trigger = 0;
+
+#endif
+
             // Request PCB Mux
 
 #if HW405292_NDD1_PCBMUX_SAVIOR
@@ -555,6 +561,8 @@ p9_cme_stop_entry()
             PK_TRACE("HW407385: Waking up the core(pm_exit=1) via SICR[4/5]");
             out32(CME_LCL_SICR_OR, core << SHIFT32(5));
 
+            CME_PM_EXIT_DELAY
+
             PK_TRACE("HW407385: Polling for core wakeup(pm_active=0) via EINR[20/21]");
 
             while((in32(CME_LCL_EINR)) & (core << SHIFT32(21)));
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
index da6a8cba..bc195d30 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
@@ -234,6 +234,8 @@ p9_cme_stop_exit_end(uint32_t core, uint32_t spwu_stop)
     PK_TRACE_PERF("SX.0A: Core Waking up(pm_exit=1) via SICR[4/5]");
     out32(CME_LCL_SICR_OR, core << SHIFT32(5));
 
+    CME_PM_EXIT_DELAY
+
     PK_TRACE_PERF("Polling for Core Waking up(pm_active=0) via EINR[20/21]");
 
     while((in32(CME_LCL_EINR)) & (core << SHIFT32(21)));
@@ -293,6 +295,13 @@ p9_cme_stop_exit_end(uint32_t core, uint32_t spwu_stop)
         out32(CME_LCL_EIPR_CLR, spwu_stop << SHIFT32(15));  // flip EIPR to falling edge
         out32(CME_LCL_SICR_OR,  spwu_stop << SHIFT32(17));  // assert spwu_done now
         G_cme_stop_record.core_in_spwu |= spwu_stop;
+
+#ifdef PCQW_ENABLE
+
+        G_cme_record.fit_record.core_quiesce_fit_trigger = 0;
+
+#endif
+
     }
 
     if ((core = (core & (~spwu_stop))))
@@ -517,6 +526,12 @@ p9_cme_stop_exit()
         out32(CME_LCL_SICR_OR,  spwu_wake << SHIFT32(17)); // assert spwu_done now
         G_cme_stop_record.core_in_spwu |= spwu_wake;
 
+#ifdef PCQW_ENABLE
+
+        G_cme_record.fit_record.core_quiesce_fit_trigger = 0;
+
+#endif
+
         if (!core)
         {
             PK_TRACE_INF("WARNING: Only Freebie Special Wakeup Processed. Return");
@@ -945,6 +960,8 @@ p9_cme_stop_exit()
             PK_TRACE_PERF("SF.RS: Self Restore Prepare, Core Waking up(pm_exit=1) via SICR[4/5]");
             out32(CME_LCL_SICR_OR, core << SHIFT32(5));
 
+            CME_PM_EXIT_DELAY
+
             PK_TRACE("Polling for core wakeup(pm_active=0) via EINR[20/21]");
 
             while((in32(CME_LCL_EINR)) & (core << SHIFT32(21)));
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
index 37026b45..e78c16a2 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_init.c
@@ -47,9 +47,6 @@ p9_cme_stop_init()
     // Basic Software Settings
     //--------------------------------------------------------------------------
 
-    // Clear SPRG0
-    ppe42_app_ctx_set(0);
-
     // Create a semaphore, updated by a single interrupt handler that services
     // two STOP threads. Thus, max sem count set to two.
 #if !defined(__IOTA__)
@@ -130,7 +127,6 @@ p9_cme_stop_init()
     bce_irr_setup();
 
 #if SKIP_BCE_SCOM_RESTORE
-    out32(CME_LCL_TSEL, BITS32(4, 4));
     ppe42_fit_setup((PkIrqHandler)bce_irr_run, 0);
 #endif
 
@@ -176,7 +172,7 @@ p9_cme_stop_init()
     //--------------------------------------------------------------------------
 
     PK_TRACE_INF("Setup: CME STOP READY");
-    out32(CME_LCL_FLAGS_OR, BIT32(CME_STOP_READY));
+    out32(CME_LCL_FLAGS_OR, BIT32(CME_FLAGS_STOP_READY));
 
 #if EPM_P9_TUNING
     asm volatile ("tw 0, 31, 0");
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_fit.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_fit.c
index fb2699ab..04bb496d 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_fit.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_fit.c
@@ -49,8 +49,6 @@ uint32_t G_aux_task_count;
 uint32_t G_tb_sync_count_threshold;
 uint32_t G_tb_sync_count;
 
-uint32_t G_last_core_quiesce_dis;
-
 extern GlobalPstateParmBlock* G_gppb;
 extern PgpeHeader_t* G_pgpe_header_data;
 extern PgpePstateRecord G_pgpe_pstate_record;
@@ -103,7 +101,6 @@ void p9_pgpe_fit_init()
     G_throttleCount = 0;
     G_beacon_count = 0;
     G_tb_sync_count = 0;
-    G_last_core_quiesce_dis = (in32(OCB_OCCFLG) & BIT32(OCCFLG_CORE_QUIESCE_WORKARND_DIS));
     ppe42_fit_setup(p9_pgpe_fit_handler, NULL);
 }
 
@@ -276,33 +273,6 @@ __attribute__((always_inline)) inline void handle_fit_timebase_sync()
     }
 }
 
-__attribute__((always_inline)) inline void handle_core_quiesce_dis()
-{
-    if(G_last_core_quiesce_dis != (in32(OCB_OCCFLG) & BIT32(OCCFLG_CORE_QUIESCE_WORKARND_DIS)))
-    {
-        G_last_core_quiesce_dis = (in32(OCB_OCCFLG) & BIT32(OCCFLG_CORE_QUIESCE_WORKARND_DIS));
-        uint64_t wrtAddr = G_last_core_quiesce_dis ? CME_SCOM_FLAGS_OR : CME_SCOM_FLAGS_CLR;
-        uint32_t q;
-        ocb_qcsr_t qcsr;
-        qcsr.value = in32(OCB_QCSR);
-
-        for (q = 0; q < MAX_QUADS; q++)
-        {
-            if (qcsr.fields.ex_config &  (0x800 >> (q << 1)))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(wrtAddr,
-                                              q, 0), BIT64(CME_CORE_QUIESCE_WORKARND_DIS));
-            }
-
-            if (qcsr.fields.ex_config &  (0x400 >> (q << 1)))
-            {
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CME(wrtAddr,
-                                              q, 1), BIT64(CME_CORE_QUIESCE_WORKARND_DIS));
-            }
-        }
-    }
-}
-
 //p9_pgpe_fit_handler
 //
 //This is a periodic FIT Handler whose period is determined
@@ -314,5 +284,4 @@ void p9_pgpe_fit_handler(void* arg, PkIrqId irq)
     handle_core_throttle();
     handle_aux_task();
     handle_fit_timebase_sync();
-    handle_core_quiesce_dis();
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
index c3a837fd..5be1b326 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
@@ -322,15 +322,6 @@ BootErrorCode_t boot_cme( uint16_t i_bootCme )
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_XIXCR,
                                               (l_cmeIndex >> 1), (l_cmeIndex % 2)), HARD_RESET_PPE);
 
-                //Set CME DIS_CORE_QUIESCE_WORKARND accordingly
-                PK_TRACE_INF("OCB_OCCFLG: %X", in32(OCB_OCCFLG));
-
-                if(in32(OCB_OCCFLG) & BIT32(OCCFLG_CORE_QUIESCE_WORKARND_DIS))
-                {
-                    GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_FLAGS_OR,
-                                                  (l_cmeIndex >> 1), (l_cmeIndex % 2)), BIT64(CME_CORE_QUIESCE_WORKARND_DIS));
-                }
-
                 // Kick off CME by giving resume command through PPE External Control Register
                 // Writing to XCR to resume PPE operation
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_XIXCR,
@@ -389,7 +380,7 @@ BootErrorCode_t boot_cme( uint16_t i_bootCme )
                     GPE_GETSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_FLAGS,
                                                   (l_cmeIndex >> 1), (l_cmeIndex % 2)), l_dataReg);
 
-                    if (!(l_dataReg & BIT64(CME_PMCR_READY)))
+                    if (!(l_dataReg & BIT64(CME_FLAGS_PMCR_READY)))
                     {
                         continue;
                     }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index cd47f029..af70aa27 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -879,21 +879,21 @@ p9_sgpe_stop_exit()
 
                 if (G_sgpe_stop_record.group.expg[qloop] & (~ex_mask))
                 {
-                    cme_flags |= BIT32(CME_SIBLING_FUNCTIONAL);
+                    cme_flags |= BIT32(CME_FLAGS_SIBLING_FUNCTIONAL);
                 }
 
                 if (ex_index == 1)
                 {
-                    cme_flags |= BIT32(CME_EX_ID);
+                    cme_flags |= BIT32(CME_FLAGS_EX_ID);
 
-                    if (!(cme_flags & BIT32(CME_SIBLING_FUNCTIONAL)))
+                    if (!(cme_flags & BIT32(CME_FLAGS_SIBLING_FUNCTIONAL)))
                     {
-                        cme_flags |= BIT32(CME_QMGR_MASTER);
+                        cme_flags |= BIT32(CME_FLAGS_QMGR_MASTER);
                     }
                 }
                 else
                 {
-                    cme_flags |= BIT32(CME_QMGR_MASTER);
+                    cme_flags |= BIT32(CME_FLAGS_QMGR_MASTER);
                 }
 
 
@@ -915,13 +915,13 @@ p9_sgpe_stop_exit()
                 {
                     if (ccsr.value & BIT32((ec_index + cloop)))
                     {
-                        cme_flags |= (BIT32(CME_CORE0_GOOD) >> cloop);
+                        cme_flags |= (BIT32(CME_FLAGS_CORE0_GOOD) >> cloop);
                         GPE_GETSCOM(GPE_SCOM_ADDR_CORE(C_NET_CTRL0,
                                                        (ec_index + cloop)), scom_data.value);
 
                         if (!(scom_data.words.upper & BIT32(18)))
                         {
-                            cme_flags |= (BIT32(CME_STOP_ENTRY_FIRST_C0) >> cloop);
+                            cme_flags |= (BIT32(CME_FLAGS_STOP_ENTRY_FIRST_C0) >> cloop);
                         }
 
 #if NIMBUS_DD_LEVEL != 10
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_init.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_init.c
index 0631087b..1c3c8d1d 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_init.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_init.c
@@ -244,6 +244,7 @@ p9_sgpe_stop_init()
     ocb_ccsr_t        ccsr       = {0};
     ocb_qcsr_t        qcsr       = {0};
     ocb_qssr_t        qssr       = {0};
+    sgpeHeader_t*     pSgpeImgHdr = (sgpeHeader_t*)(OCC_SRAM_SGPE_HEADER_ADDR);
 
     //--------------------------------------------------------------------------
     // Basic Software Settings
@@ -302,16 +303,6 @@ p9_sgpe_stop_init()
 
     G_sgpe_stop_record.group.core[VECTOR_PCWU]   = 0;
 
-    //Write Core Quiesce Workaround Disable bit to OCC Flag
-    uint32_t occflg = in32(OCB_OCCFLG) & ~BIT32(13);
-    sgpeHeader_t* pSgpeImgHdr = (sgpeHeader_t*)(OCC_SRAM_SGPE_HEADER_ADDR);
-
-    if(pSgpeImgHdr->g_sgpe_reserve_flags & BIT32(7))//SGPE_CORE_PERIODIC_QUIESCE_DISABLE_POS))
-    {
-        out32(OCB_OCCFLG, occflg | BIT32(OCCFLG_CORE_QUIESCE_WORKARND_DIS));
-    }
-
-
     for(qloop = 0, m_1c = BIT32(0), m_2c = BITS32(0, 2), m_4c = BITS32(0, 4),
         m_1x = BIT32(0), m_2x = BITS32(0, 2), m_qs = BIT32(14);
         qloop < MAX_QUADS;
@@ -341,6 +332,7 @@ p9_sgpe_stop_init()
                     G_sgpe_stop_record.group.core[VECTOR_ACTIVE] |= m_1c;
                     G_sgpe_stop_record.state[qloop].cme_flags |= (0x80 >> cloop);
                     G_sgpe_stop_record.level[qloop][cloop] = STOP_LEVEL_0;
+
                 }
             }
         }
@@ -483,22 +475,22 @@ p9_sgpe_stop_init()
 
                 if (cmeBootList & BIT16(((qloop << 1) + (xloop ^ 1))))
                 {
-                    cme_flags |= BIT32(CME_SIBLING_FUNCTIONAL);
+                    cme_flags |= BIT32(CME_FLAGS_SIBLING_FUNCTIONAL);
                 }
 
                 if (xloop)
                 {
-                    cme_flags |= BIT32(CME_EX_ID);
+                    cme_flags |= BIT32(CME_FLAGS_EX_ID);
 
-                    if(!(cme_flags & BIT32(CME_SIBLING_FUNCTIONAL)))
+                    if(!(cme_flags & BIT32(CME_FLAGS_SIBLING_FUNCTIONAL)))
                     {
                         //EX1 is configured, but EX0 isn't
-                        cme_flags  |= BIT32(CME_QMGR_MASTER);
+                        cme_flags  |= BIT32(CME_FLAGS_QMGR_MASTER);
                     }
                 }
                 else
                 {
-                    cme_flags |= BIT32(CME_QMGR_MASTER);
+                    cme_flags |= BIT32(CME_FLAGS_QMGR_MASTER);
                 }
 
                 PK_TRACE_DBG("CME%d%d_FLAGS :%x", qloop, xloop, cme_flags);