PM: Implement L2 Resclk Function

Change-Id: I0efbab5defe2ffbd5c5fe86690c6e7a498f3c4d1
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/43331
Reviewed-by: YUE DU <daviddu@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Reviewed-by: BRIAN D. VICTOR <brian.d.victor1@ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

11 files changed, 369 insertions, 87 deletions

diff --git a/import/chips/p9/common/pmlib/include/stop_sgpe_cme_api.h b/import/chips/p9/common/pmlib/include/stop_sgpe_cme_api.h
index fb7fb201..eed0399f 100644
--- a/import/chips/p9/common/pmlib/include/stop_sgpe_cme_api.h
+++ b/import/chips/p9/common/pmlib/include/stop_sgpe_cme_api.h
@@ -28,6 +28,18 @@
 
 enum SGPE_STOP_IRQ_PAYLOAD_MASKS
 {
+    TYPE2_PAYLOAD_STOP_LEVEL               = 0x00F,
+    TYPE2_PAYLOAD_ENTRY_EVENT              = 0x000, // 0000
+    TYPE2_PAYLOAD_ENTRY_RCLK               = 0x100, // 0001
+    // 0010 reserved
+    // 0011 reserved
+
+    TYPE2_PAYLOAD_EXIT_EVENT               = 0xC00, // 1100
+    TYPE2_PAYLOAD_HARDWARE_WAKEUP          = 0x800, // 1000
+    TYPE2_PAYLOAD_SOFTWARE_WAKEUP          = 0x400, // 0100
+    TYPE2_PAYLOAD_EXIT_RCLK                = 0x500, // 0101
+    // 0110 reserved
+    TYPE2_PAYLOAD_DECREMENTER_WAKEUP       = 0x700, // 0111
 
     TYPE2_PAYLOAD_SUSPEND_OP_MASK          = 0x400,
     TYPE2_PAYLOAD_SUSPEND_EXIT_MASK        = 0x200,
@@ -35,10 +47,6 @@ enum SGPE_STOP_IRQ_PAYLOAD_MASKS
     TYPE2_PAYLOAD_SUSPEND_BOTH_MASK        = 0x300,
     TYPE2_PAYLOAD_SUSPEND_ACK_MASK         = 0x080,
 
-    TYPE2_PAYLOAD_HARDWARE_WAKEUP          = 0x800,
-    TYPE2_PAYLOAD_EXIT_EVENT               = 0xC00,
-    TYPE2_PAYLOAD_STOP_LEVEL               = 0x00F,
-    TYPE3_PAYLOAD_EXIT_EVENT               = 0xC00,
     TYPE6_PAYLOAD_EXIT_EVENT               = 0x00F
 };
 
@@ -56,10 +64,15 @@ enum CME_DOORBELL_MESSAGE_IDS
     MSGID_DB1_UNSUSPEND_STOP_ENTRIES       = 0x01,
     MSGID_DB1_UNSUSPEND_STOP_EXITS         = 0x02,
     MSGID_DB1_UNSUSPEND_STOP_ENTRIES_EXITS = 0x03,
-    MSGID_DB1_WAKEUP_GRANTED               = 0x04,
+    // 0x04 Illegal
     MSGID_DB1_SUSPEND_STOP_ENTRIES         = 0x05,
     MSGID_DB1_SUSPEND_STOP_EXITS           = 0x06,
-    MSGID_DB1_SUSPEND_STOP_ENTRIES_EXITS   = 0x07
+    MSGID_DB1_SUSPEND_STOP_ENTRIES_EXITS   = 0x07,
+    MSGID_DB1_WAKEUP_GRANTED               = 0x08,
+
+    MSGID_DB2_DECREMENTER_WAKEUP           = 0x01,
+    MSGID_DB2_RESONANT_CLOCK_DISABLE       = 0x02,
+    MSGID_DB2_RESONANT_CLOCK_ENABLE        = 0x03
 };
 
 #endif  /* __STOP_SGPE_CME_API_H__ */
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
index e326115d..68c14342 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
@@ -108,9 +108,7 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
 
     out32_sh(CME_LCL_EISR_CLR, G_cme_record.core_enabled << 25);//Clear DB0_C0/C1
 
-    out32(CME_LCL_ICCR_OR, BIT32(5));//Send Direct InterCME_IN0(Ack to QM-CME)
-    out32(CME_LCL_ICCR_CLR, BIT32(5));//Clear Ack
-    out32(CME_LCL_EISR_CLR, BIT32(7));//Clear InterCME_IN0
+    intercme_direct(INTERCME_DIRECT_IN0, INTERCME_DIRECT_ACK);
 
     pk_irq_vec_restore(&ctx);
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
index 9de69983..760a37dd 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
@@ -150,7 +150,6 @@ void intercme_msg_send(uint32_t msg, INTERCME_MSG_TYPE type)
     out32(CME_LCL_ICSR, (msg << 4) | type);
 
     PK_TRACE_DBG("imt send | msg=%08x", ((msg << 4) | type));
-    PK_TRACE_DBG("buff");
 
     // Block on ack from companion CME
     while(!(in32(CME_LCL_EISR) & BIT32(30))) {}
@@ -163,9 +162,7 @@ void intercme_msg_recv(uint32_t* msg, INTERCME_MSG_TYPE type)
     // Poll for inter-cme communication from QM
     while(!(in32(CME_LCL_EISR) & BIT32(29))) {}
 
-    // Get the initial pstate value
     *msg = in32(CME_LCL_ICRR);
-
     PK_TRACE_DBG("imt recv | msg=%08x", *msg);
 
     if(*msg & type)
@@ -185,6 +182,46 @@ void intercme_msg_recv(uint32_t* msg, INTERCME_MSG_TYPE type)
     out32(CME_LCL_EISR_CLR, BIT32(29));
 }
 
+void intercme_direct(INTERCME_DIRECT_INTF intf, INTERCME_DIRECT_TYPE type)
+{
+    uint32_t addr_offset = 0;
+    // Send intercme interrupt, this is the same whether notifying or acking
+    out32(CME_LCL_ICCR_OR , BIT32(intf));
+    out32(CME_LCL_ICCR_CLR, BIT32(intf));
+
+    // Adjust the EI*R base address based on which intercme direct interface
+    // is used since the bits are spread across both words in the EI*R registers
+    if(intf == INTERCME_DIRECT_IN0)
+    {
+        // IN0: ICCR[5], EI*R[7]
+        intf += 2;
+    }
+    else
+    {
+        // IN1: ICCR[6], EI*R[38], ie. second half EI*R[6]
+        // IN2: ICCR[7], EI*R[39], ie. second half EI*R[7]
+        addr_offset = 4;
+    }
+
+    if(type == INTERCME_DIRECT_NOTIFY)
+    {
+        uint32_t intercme_acked = 0;
+#if SIMICS_TUNING == 1
+        intercme_acked = 1;
+#endif
+
+        while(!intercme_acked)
+        {
+            if(in32((CME_LCL_EISR + addr_offset)) & BIT32(intf))
+            {
+                intercme_acked = 1;
+            }
+        }
+    }
+
+    out32((CME_LCL_EISR_CLR + addr_offset), BIT32(intf)); // Clear the interrupt
+}
+
 #ifdef USE_CME_RESCLK_FEATURE
 uint32_t p9_cme_resclk_get_index(uint32_t pstate)
 {
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
index 00763f51..44cb61a0 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
@@ -132,6 +132,19 @@ typedef enum
     IMT_SYNC_SIBLING = (uint32_t)0x00000003
 } INTERCME_MSG_TYPE;
 
+typedef enum
+{
+    INTERCME_DIRECT_IN0 = 5,
+    INTERCME_DIRECT_IN1 = 6,
+    INTERCME_DIRECT_IN2 = 7
+} INTERCME_DIRECT_INTF;
+
+typedef enum
+{
+    INTERCME_DIRECT_NOTIFY = 0,
+    INTERCME_DIRECT_ACK    = 1
+} INTERCME_DIRECT_TYPE;
+
 typedef struct
 {
 #if !defined(__IOTA__)
@@ -167,6 +180,7 @@ void ippm_read(uint32_t addr, uint64_t* data);
 void ippm_write(uint32_t addr, uint64_t data);
 void intercme_msg_send(uint32_t msg, INTERCME_MSG_TYPE type);
 void intercme_msg_recv(uint32_t* msg, INTERCME_MSG_TYPE type);
+void intercme_direct(INTERCME_DIRECT_INTF intf, INTERCME_DIRECT_TYPE type);
 void p9_cme_analog_control(uint32_t core_mask, ANALOG_CONTROL enable);
 void p9_cme_pstate_pmsr_updt(uint32_t coreMask);
 #ifdef USE_CME_RESCLK_FEATURE
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
index b6bcc4c9..e599d336 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
@@ -245,6 +245,7 @@ void p9_cme_pstate_db_thread(void* arg)
         PK_PANIC(CME_PSTATE_RESCLK_ENABLED_AT_BOOT);
     }
 
+#if NIMBUS_DD_LEVEL >= 21 || CUMULUS_DD_LEVEL > 10
     ippm_read(QPPM_EXCGCR, &scom_data);
     // Ignore clk_sync_enable, clkglm_async_reset, clkglm_sel, and reserved
     scom_data &= ~(BITS64(29, 9) | BITS64(42, 22));
@@ -254,6 +255,8 @@ void p9_cme_pstate_db_thread(void* arg)
         PK_PANIC(CME_PSTATE_RESCLK_ENABLED_AT_BOOT);
     }
 
+#endif
+
 #ifdef USE_CME_RESCLK_FEATURE
 
     // Resonant Clocking Initialization (QM + Sibling)
@@ -501,33 +504,12 @@ inline void p9_cme_pstate_db0_suspend()
 
 inline void p9_cme_pstate_notify_sib()
 {
-    uint32_t intercme_acked = 0;
-    uint32_t eisr;
-
     PK_TRACE_INF("DB_TH: Notify Enter\n");
 
     //Notify sibling CME(if any)
-    if (G_cme_pstate_record.siblingCMEFlag)
+    if(G_cme_pstate_record.siblingCMEFlag)
     {
-        //Send interCME interrupt
-        out32(CME_LCL_ICCR_OR, BIT32(5)); //Send direct InterCME_IN0
-        out32(CME_LCL_ICCR_CLR, BIT32(5));//Clear
-
-#if SIMICS_TUNING == 1
-        intercme_acked = 1;
-#endif
-
-        while (!intercme_acked)
-        {
-            eisr = in32(CME_LCL_EISR);
-
-            if (eisr & 0x01000000)
-            {
-                intercme_acked = 1;
-            }
-        }
-
-        out32(CME_LCL_EISR_CLR, BIT32(7));//Clear InterCME_IN0
+        intercme_direct(INTERCME_DIRECT_IN0, INTERCME_DIRECT_ACK);
     }
 
     PK_TRACE_INF("DB_TH: Notify Exit\n");
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
index b387d73d..b54bb458 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
@@ -26,9 +26,11 @@
 #include "p9_cme_stop.h"
 #include "p9_cme_stop_enter_marks.h"
 #include "p9_cme_irq.h"
+#include "p9_cme_pstate.h"
 
-extern CmeStopRecord G_cme_stop_record;
-extern CmeRecord     G_cme_record;
+extern CmeStopRecord   G_cme_stop_record;
+extern CmePstateRecord G_cme_pstate_record;
+extern CmeRecord       G_cme_record;
 
 
 
@@ -60,7 +62,7 @@ p9_cme_stop_pcwu_handler(void* arg, PkIrqId irq)
                 if (!(scom_data.words.upper & BIT32(10)))
                 {
                     pig.fields.req_intr_type    = PIG_TYPE2;
-                    pig.fields.req_intr_payload = 0x400;
+                    pig.fields.req_intr_payload = TYPE2_PAYLOAD_DECREMENTER_WAKEUP;
                     CME_PUTSCOM_NOP(PPM_PIG, core_mask, pig.value);
                 }
 
@@ -94,6 +96,7 @@ p9_cme_stop_pcwu_handler(void* arg, PkIrqId irq)
 }
 
 
+
 // When take an Interrupt on falling edge of SPWU from a CPPM.
 // 1) Read EINR to check if another one has been set
 //    in the meantime from the same core.  If so abort.
@@ -234,27 +237,96 @@ p9_cme_stop_enter_handler(void* arg, PkIrqId irq)
 #endif
 }
 
+
+
 void
 p9_cme_stop_db2_handler(void* arg, PkIrqId irq)
 {
     PkMachineContext ctx __attribute__((unused));
+    cppm_cmedb2_t    db2 = {0};
+    ppm_pig_t        pig = {0};
 
     MARK_TRAP(STOP_DB2_HANDLER)
     PK_TRACE_DBG("DB2 Handler Trigger %d", irq);
 
     // read and clear doorbell
     uint32_t core = (in32(CME_LCL_EISR) & BITS32(18, 2)) >> SHIFT32(19);
+    CME_GETSCOM(CPPM_CMEDB2, core, db2.value);
     CME_PUTSCOM_NOP(CPPM_CMEDB2, core, 0);
     out32(CME_LCL_EISR_CLR, (core << SHIFT32(19)));
 
-    // unmask pc interrupt pending to wakeup that is still pending
-    core &= (~(G_cme_stop_record.core_running));
-    G_cme_stop_record.core_blockpc &= ~core;
-    g_eimr_override &= ~(((uint64_t)core) << SHIFT64(13));
+    switch (db2.fields.cme_message_numbern)
+    {
+        case MSGID_DB2_DECREMENTER_WAKEUP:
+
+            // unmask pc interrupt pending to wakeup that is still pending
+            core &= (~(G_cme_stop_record.core_running));
+            G_cme_stop_record.core_blockpc &= ~core;
+            g_eimr_override &= ~(((uint64_t)core) << SHIFT64(13));
+            break;
+
+        case MSGID_DB2_RESONANT_CLOCK_DISABLE:
+
+#if (NIMBUS_DD_LEVEL < 21 || CUMULUS_DD_LEVEL == 10) && DISABLE_STOP8 == 1
+#ifdef USE_CME_RESCLK_FEATURE
+
+            // Quad going into Stop11, need to potentially disable Resclks
+            if((in32(CME_LCL_FLAGS) & BIT32(CME_FLAGS_RCLK_OPERABLE))
+               && G_cme_pstate_record.qmFlag)
+            {
+                PkMachineContext ctx;
+                pk_critical_section_enter(&ctx);
+
+                p9_cme_resclk_update(ANALOG_COMMON, ANALOG_PSTATE_RESCLK_OFF,
+                                     G_cme_pstate_record.resclkData.common_resclk_idx);
+
+                pk_critical_section_exit(&ctx);
+            }
+
+#endif
+#endif
+            // Finish handshake with SGPE for Stop11 via PIG
+            pig.fields.req_intr_type    = PIG_TYPE3;
+            pig.fields.req_intr_payload = TYPE2_PAYLOAD_ENTRY_RCLK | STOP_LEVEL_11;
+            CME_PUTSCOM_NOP(PPM_PIG, core, pig.value);
+            break;
+
+        case MSGID_DB2_RESONANT_CLOCK_ENABLE:
+
+#if (NIMBUS_DD_LEVEL < 21 || CUMULUS_DD_LEVEL == 10) && DISABLE_STOP8 == 1
+#ifdef USE_CME_RESCLK_FEATURE
+
+            // Quad aborted Stop11, need to regressively enable Resclks
+            // IF wakeup from fully entered Stop11, this is done by QM
+            if((in32(CME_LCL_FLAGS) & BIT32(CME_FLAGS_RCLK_OPERABLE))
+               && G_cme_pstate_record.qmFlag)
+            {
+                PkMachineContext ctx;
+                pk_critical_section_enter(&ctx);
+
+                p9_cme_resclk_update(ANALOG_COMMON, G_cme_pstate_record.quadPstate,
+                                     G_cme_pstate_record.resclkData.common_resclk_idx);
+
+                pk_critical_section_exit(&ctx);
+            }
+
+#endif
+#endif
+            // Finish handshake with SGPE for Stop11 via PIG
+            pig.fields.req_intr_type    = PIG_TYPE3;
+            pig.fields.req_intr_payload = TYPE2_PAYLOAD_EXIT_RCLK;
+            CME_PUTSCOM_NOP(PPM_PIG, core, pig.value);
+            break;
+
+        default:
+            break;
+    }
 
     pk_irq_vec_restore(&ctx);
 }
 
+
+
 void
 p9_cme_stop_db1_handler(void* arg, PkIrqId irq)
 {
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
index 98a2d8ec..d8503a92 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
@@ -385,6 +385,22 @@ BootErrorCode_t boot_cme( uint16_t i_bootCme )
                         continue;
                     }
 
+#if (NIMBUS_DD_LEVEL < 21 || CUMULUS_DD_LEVEL == 10) && DISABLE_STOP8 == 1
+
+                    if(l_dataReg & BIT64(CME_FLAGS_RCLK_OPERABLE))
+                    {
+                        // Set the EXCGCR to point to QACCR for L2 Resonance
+                        uint64_t excgcr;
+                        GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(QPPM_EXCGCR, (l_cmeIndex / 2)), excgcr);
+                        // EX0: bits 38,40; EX1: bits 39,41
+                        excgcr |= BIT64(38 + (l_cmeIndex % 2)) | BIT64(40 + (l_cmeIndex % 2));
+                        PK_TRACE("Setting EXCGCR[ex=%d] to point to QACCR", l_cmeIndex);
+                        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_EXCGCR, (l_cmeIndex / 2)), excgcr);
+                    }
+
+                    // TODO #else
+#endif
+
                     l_cmeRdyCnt++;
                     cmeReadyList = cmeReadyList | l_cmeActiveBit;
                 }//end for
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index b082eb6c..cf8ec70e 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -222,15 +222,17 @@ enum SGPE_SUSPEND_FUNCTION_STATUS
 
 enum SGPE_STOP_VECTOR_INDEX
 {
-    VECTOR_BLOCKE                     = 0, //(core_save, quad_req, qswu_save, qex01)
-    VECTOR_BLOCKX                     = 1, //(core_save, quad_req, qswu_save, qex01)
-    VECTOR_ENTRY                      = 2, //(core_ipc,  quad,     qswu)
-    VECTOR_EXIT                       = 3, //(core,      quad_ipc  qswu)
-    VECTOR_ACTIVE                     = 4, //(core_ipc,  quad_ipc, qswu_active)
-    VECTOR_CONFIG                     = 5, //(core,      quad)
-    VECTOR_PCWU                       = 6, //(core)
-    VECTOR_PIGE                       = 7, //(core)
-    VECTOR_PIGX                       = 8, //(core)
+    VECTOR_BLOCKE                     = 0, //(core_save,   quad_req, qswu_save, qex01)
+    VECTOR_BLOCKX                     = 1, //(core_save,   quad_req, qswu_save, qex01)
+    VECTOR_ENTRY                      = 2, //(core_ipc,    quad,     qswu)
+    VECTOR_EXIT                       = 3, //(core,        quad_ipc  qswu)
+    VECTOR_ACTIVE                     = 4, //(core_ipc,    quad_ipc, qswu_active)
+    VECTOR_CONFIG                     = 5, //(core,        quad)
+    VECTOR_RCLKE                      = 6, //(core_blocke, quad)
+    VECTOR_RCLKX                      = 7, //(core_blockx, quad)
+    VECTOR_PIGE                       = 8, //(core)
+    VECTOR_PIGX                       = 9, //(core)
+    VECTOR_PCWU                       = 10,//(core)
 };
 
 typedef struct
@@ -254,8 +256,8 @@ typedef struct
     uint32_t qex0[2]; // 6  bits
     uint32_t qex1[2]; // 6  bits
     uint32_t qswu[5]; // 6  bits
-    uint32_t quad[6]; // 6  bits
-    uint32_t core[9]; // 24 bits
+    uint32_t quad[8]; // 6  bits
+    uint32_t core[11];// 24 bits
 } sgpe_group_t;
 
 typedef struct
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index 7c03ea15..d1ac193a 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -53,6 +53,7 @@ p9_sgpe_stop_entry()
     uint64_t     local_xstop       = 0;
     data64_t     scom_data         = {0};
     data64_t     temp_data         = {0};
+    ppm_pig_t    pig               = {0};
 #if HW386311_NDD1_PBIE_RW_PTR_STOP11_FIX
     uint32_t      spin             = 0;
 #endif
@@ -79,7 +80,8 @@ p9_sgpe_stop_entry()
 
     for(qloop = 0; qloop < MAX_QUADS; qloop++)
     {
-        if (G_sgpe_stop_record.group.qswu[VECTOR_ACTIVE] & BIT32(qloop))
+        if ((G_sgpe_stop_record.group.qswu[VECTOR_ACTIVE] |
+             (~G_sgpe_stop_record.group.quad[VECTOR_CONFIG])) & BIT32(qloop))
         {
             continue;
         }
@@ -121,20 +123,74 @@ p9_sgpe_stop_entry()
         if(G_sgpe_stop_record.state[qloop].act_state_q <  LEVEL_EQ_BASE &&
            G_sgpe_stop_record.state[qloop].req_state_q >= LEVEL_EQ_BASE)
         {
-            G_sgpe_stop_record.group.quad[VECTOR_ENTRY] |= BIT32(qloop);
+            // if resonant clock disable is completed, process stop11 entry
+            if (G_sgpe_stop_record.group.quad[VECTOR_RCLKE] & BIT32(qloop))
+            {
+                G_sgpe_stop_record.group.quad[VECTOR_RCLKE] &= ~BIT32(qloop);
+
+                // if during resonant clock disable, any exit occured, re-assert them,
+                // but we are going to complete the stop11 entry prior to process it
+                for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
+                {
+                    cindex = (qloop << 2) + cloop;
+
+                    if (G_sgpe_stop_record.group.core[VECTOR_RCLKE] & BIT32(cindex))
+                    {
+                        G_sgpe_stop_record.group.core[VECTOR_RCLKE] &= ~BIT32(cindex);
+                        pig.fields.req_intr_payload = TYPE2_PAYLOAD_SOFTWARE_WAKEUP;
+                        pig.fields.req_intr_type = PIG_TYPE3;
+                        GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(PPM_PIG, cindex), pig.value);
+                    }
+                }
+
+                G_sgpe_stop_record.group.quad[VECTOR_ENTRY] |= BIT32(qloop);
 
 #if DISABLE_STOP8
 
-            ocb_qssr_t qssr = {0};
-            qssr.value      = in32(OCB_QSSR);
+                ocb_qssr_t qssr = {0};
+                qssr.value      = in32(OCB_QSSR);
+
+                // check qssr for already stopped ex
+                G_sgpe_stop_record.group.ex01[qloop] =
+                    (((~qssr.value) & BITS32((qloop << 1), 2)) >>
+                     SHIFT32(((qloop << 1) + 1)));
+
+#endif
+
+            }
+            // if stop11 entry qualifies, hold on processing it but first
+            // send DB to Quad-Manager to disable the resonant clock
+            else
+            {
+                // assume ex0 core0 is good
+                cindex = (qloop << 2);
+
+                // if ex0 is bad, switch to ex1
+                if (!(G_sgpe_stop_record.group.expg[qloop] & FST_EX_IN_QUAD))
+                {
+                    cindex += 2;
+                }
+
+                // if first core in the ex is bad
+                if (!(G_sgpe_stop_record.group.core[VECTOR_CONFIG] & BIT32(cindex)))
+                {
+                    cindex++;
+                }
+
+                // send DB2 with msgid 0x2 to the first good core
+                // to trigger Quad Manager to disable resonant clock
 
-            // check qssr for already stopped ex
-            G_sgpe_stop_record.group.ex01[qloop] =
-                (((~qssr.value) & BITS32((qloop << 1), 2)) >>
-                 SHIFT32(((qloop << 1) + 1)));
+#if NIMBUS_DD_LEVEL != 10
+
+                GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB2, cindex), BIT64(6));
+
+#else
+
+                p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB2, cindex), BIT32(6));
 
 #endif
 
+            }
         }
 
         G_sgpe_stop_record.group.ex01[qloop] &=
@@ -161,9 +217,6 @@ p9_sgpe_stop_entry()
         }
     }
 
-    G_sgpe_stop_record.group.quad[VECTOR_ENTRY] &=
-        G_sgpe_stop_record.group.quad[VECTOR_CONFIG];
-
     PK_TRACE_DBG("Entry Vectors: Q0_EX[%x] Q1_EX[%x] Q2_EX[%x] QSPWU[%x]",
                  G_sgpe_stop_record.group.ex01[0],
                  G_sgpe_stop_record.group.ex01[1],
@@ -599,7 +652,7 @@ p9_sgpe_stop_entry()
                         cindex = (qloop << 2) + cloop;
 
                         if ((in32(OCB_OPIT2CN(cindex)) & TYPE2_PAYLOAD_EXIT_EVENT) ||
-                            (in32(OCB_OPIT3CN(cindex)) & TYPE3_PAYLOAD_EXIT_EVENT))
+                            (in32(OCB_OPIT3CN(cindex)) & TYPE2_PAYLOAD_EXIT_EVENT))
                         {
                             PK_TRACE_DBG("Abort: core wakeup detected");
                             l3_purge_aborted = 1;
@@ -703,6 +756,37 @@ p9_sgpe_stop_entry()
                 PK_PANIC(SGPE_STOP_EXIT_DROP_SLV_LOCK_FAILED);
             }
 
+            // assume ex0 core0 is good
+            cindex = (qloop << 2);
+
+            // if ex0 is bad, switch to ex1
+            if (!(G_sgpe_stop_record.group.expg[qloop] & FST_EX_IN_QUAD))
+            {
+                cindex += 2;
+            }
+
+            // if first core in the ex is bad, switch to second core
+            if (!(G_sgpe_stop_record.group.core[VECTOR_CONFIG] & BIT32(cindex)))
+            {
+                cindex++;
+            }
+
+            // send DB2 with msgid 0x3 to the first good core
+            // to trigger Quad Manager to enable resonant clock again
+
+#if NIMBUS_DD_LEVEL != 10
+
+            GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB2, cindex), BITS64(6, 2));
+
+#else
+
+            p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB2, cindex), BITS32(6, 2));
+
+#endif
+
+            // block handoff to cme until resonant clock enable is completed.
+            G_sgpe_stop_record.group.quad[VECTOR_RCLKX] |= BIT32(qloop);
+
             // For IPC reporting, taking aborted quad out of the list
             G_sgpe_stop_record.group.quad[VECTOR_ENTRY] &= ~BIT32(qloop);
             continue;
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index 1dd7395b..baaa72e5 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -222,7 +222,15 @@ p9_sgpe_stop_exit_end(uint32_t qloop)
 
         if (G_sgpe_stop_record.group.core[VECTOR_EXIT] & BIT32(cindex))
         {
-            p9_sgpe_stop_exit_handoff_cme(cindex);
+            if (G_sgpe_stop_record.group.quad[VECTOR_RCLKX] & BIT32(qloop))
+            {
+                G_sgpe_stop_record.group.core[VECTOR_RCLKX] |= BIT32(cindex);
+            }
+            else
+            {
+                G_sgpe_stop_record.group.core[VECTOR_RCLKX] &= ~BIT32(cindex);
+                p9_sgpe_stop_exit_handoff_cme(cindex);
+            }
         }
 
     }
@@ -251,7 +259,8 @@ p9_sgpe_stop_exit_handoff_cme(uint32_t cindex)
     // if waken up by pc, send doorbell to unmask pc
     if (G_sgpe_stop_record.group.core[VECTOR_PCWU] & BIT32(cindex))
     {
-
+        // send DB2 with msgid 0x1 to the core request wakeup via decrementor to
+        // alert CME to process via unmask PC_INTR_PENDING in addition to handoff
         PK_TRACE_DBG("SX.CME: Core[%d] Waking Up by PC_Interrupt_Pending", cindex);
 
 #if NIMBUS_DD_LEVEL != 10
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
index 1eef2b7a..b7fcfb9d 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_irq_handlers.c
@@ -52,8 +52,8 @@ SgpeStopRecord G_sgpe_stop_record __attribute__((section (".dump_ptrs"))) =
         {0, 0},
         {0, 0},
         {0, 0, 0, 0, 0},
-        {0, 0, 0, 0, 0, 0},
-        {0, 0, 0, 0, 0, 0, 0, 0, 0}
+        {0, 0, 0, 0, 0, 0, 0, 0},
+        {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
     },
     // wof status
     {0, 0, 0},
@@ -94,7 +94,17 @@ p9_sgpe_stop_suspend_db1_cme(uint32_t qloop, uint32_t msgid)
         {
             scom_data.words.upper = msgid;
             scom_data.words.lower = 0;
+
+#if NIMBUS_DD_LEVEL != 10
+
+            GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB1, cindex), scom_data.value);
+
+#else
+
             p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB1, cindex), scom_data.value);
+
+#endif
+
         }
         // otherwise send an ack pig on behalf of that quad(stop11 or partial bad) or ex (ex is partial bad)
         else
@@ -105,7 +115,7 @@ p9_sgpe_stop_suspend_db1_cme(uint32_t qloop, uint32_t msgid)
             }
 
             pig.fields.req_intr_payload = msgid >> 16;
-            pig.fields.req_intr_payload |= 0x080; // set bit 4 for ack package
+            pig.fields.req_intr_payload |= TYPE2_PAYLOAD_SUSPEND_ACK_MASK;
             pig.fields.req_intr_type    = PIG_TYPE2;
             GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(PPM_PIG, cindex), pig.value);
         }
@@ -445,26 +455,64 @@ p9_sgpe_pig_type23_parser(const uint32_t type)
                 }
                 else
                 {
+                    // if wakeup by pc_intr_pending,
+                    // go exit with flag to do extra doorbell from normal wakeup
+                    if (cpayload == TYPE2_PAYLOAD_DECREMENTER_WAKEUP)
+                    {
+                        PK_TRACE_INF("Core Request Exit with Decrementer Wakeup");
+                        G_sgpe_stop_record.group.core[VECTOR_PCWU] |= BIT32(cindex);
+                    }
+
+                    // Quad-Manager completed the resonant clock enable, proceed stop5 exit
+                    if (cpayload == TYPE2_PAYLOAD_EXIT_RCLK)
+                    {
+                        PK_TRACE_INF("Core Request Exit Allowed as Resonant Clock Enable is Completed");
+                        G_sgpe_stop_record.group.quad[VECTOR_RCLKX] &= ~BIT32(qloop);
+                    }
+
                     if (G_sgpe_stop_record.group.quad[VECTOR_BLOCKX] & BIT32(qloop))
                     {
-                        PK_TRACE_DBG("Core Request Exit, but in Block Wakeup Mode so Ignore");
-                        G_sgpe_stop_record.group.core[VECTOR_BLOCKX] |= BIT32(cindex);
+                        PK_TRACE_DBG("Core Request Exit But in Block Wakeup Mode so Ignore");
+
+                        if (cpayload == TYPE2_PAYLOAD_EXIT_RCLK)
+                        {
+                            G_sgpe_stop_record.group.core[VECTOR_BLOCKX] |=
+                                G_sgpe_stop_record.group.core[VECTOR_RCLKX];
+                        }
+                        else
+                        {
+                            G_sgpe_stop_record.group.core[VECTOR_BLOCKX] |= BIT32(cindex);
+                        }
                     }
-                    else
+                    // FIXME exit unblock above can bypass rclk exit block
+                    else if (G_sgpe_stop_record.group.quad[VECTOR_RCLKE] & BIT32(qloop))
                     {
-                        PK_TRACE_INF("Core Request Exit");
-                        G_sgpe_stop_record.group.core[VECTOR_PIGX] |= BIT32(cindex);
+                        if (cpayload == TYPE2_PAYLOAD_EXIT_RCLK)
+                        {
+                            PK_TRACE_ERR("ERROR: IMPOSSIBLE! RCLK PROTOCOL BROKEN! HALT SGPE!");
+                            pk_halt();
+                        }
 
-                        PK_TRACE("Update STOP history on core: in transition of exit");
-                        scom_data.words.upper = SSH_EXIT_IN_SESSION;
-                        scom_data.words.lower = 0;
-                        GPE_PUTSCOM_VAR(PPM_SSHSRC, CORE_ADDR_BASE, cindex, 0, scom_data.value);
+                        PK_TRACE_INF("Core Request Exit But Resonent Clock Disable Ongoing so Ignore");
+                        G_sgpe_stop_record.group.core[VECTOR_RCLKE] |= BIT32(cindex);
+                    }
+                    else
+                    {
+                        PK_TRACE_INF("Core Request Exit Confirmed");
 
-                        // if wakeup by pc_intr_pending,
-                        // go exit with flag to do extra doorbell from normal wakeup
-                        if (cpayload == 0x400)
+                        if (cpayload == TYPE2_PAYLOAD_EXIT_RCLK)
+                        {
+                            G_sgpe_stop_record.group.core[VECTOR_PIGX] |=
+                                G_sgpe_stop_record.group.core[VECTOR_RCLKX];
+                        }
+                        else
                         {
-                            G_sgpe_stop_record.group.core[VECTOR_PCWU] |= BIT32(cindex);
+                            G_sgpe_stop_record.group.core[VECTOR_PIGX] |= BIT32(cindex);
+
+                            PK_TRACE("Update STOP history on core: in transition of exit");
+                            scom_data.words.upper = SSH_EXIT_IN_SESSION;
+                            scom_data.words.lower = 0;
+                            GPE_PUTSCOM_VAR(PPM_SSHSRC, CORE_ADDR_BASE, cindex, 0, scom_data.value);
                         }
                     }
                 }
@@ -483,19 +531,26 @@ p9_sgpe_pig_type23_parser(const uint32_t type)
 
                 if (!timeout)
                 {
-                    PK_TRACE_ERR("ERROR: Received Phantom Entry PIG \
-                                  When Wakeup_notify_select = 0. HALT SGPE!");
+                    PK_TRACE_ERR("ERROR: Received Phantom Entry PIG"
+                                 " When Wakeup_notify_select = 0. HALT SGPE!");
                     PK_PANIC(SGPE_PIG_TYPE23_ENTRY_WNS_CME);
                 }
 
+                // Quad-Manager completed the resonant clock disable, proceed stop11 entry
+                if (cpayload == (TYPE2_PAYLOAD_ENTRY_RCLK | STOP_LEVEL_11))
+                {
+                    PK_TRACE_INF("Core Request Entry Allowed as Resonant Clock Disable is Completed");
+                    G_sgpe_stop_record.group.quad[VECTOR_RCLKE] |= BIT32(qloop);
+                }
+
                 if (G_sgpe_stop_record.group.quad[VECTOR_BLOCKE] & BIT32(qloop))
                 {
-                    PK_TRACE_DBG("Core Request Entry, but in Block Entry Mode so Ignore");
+                    PK_TRACE_DBG("Core Request Entry But in Block Entry Mode so Ignore");
                     G_sgpe_stop_record.group.core[VECTOR_BLOCKE] |= BIT32(cindex);
                 }
                 else
                 {
-                    PK_TRACE_INF("Core Request Entry");
+                    PK_TRACE_INF("Core Request Entry Confirmed");
                     G_sgpe_stop_record.group.core[VECTOR_PIGE] |= BIT32(cindex);
                 }
             }