Pstate: Resonant Clocking Enablement - CME Hcode

- implement p9_cme_resclk_control and p9_cme_resclk_update
- initialize resclk datastructures during init
- add handler for comm_recvd (intercme msg interrupt) and corresponding
  irq priority (10)
- update p9_cme_qm_flags.h to match header definition (uint16_t) and fix
  duplicate flag definition
- add intercme msg functions and packet definition

Change-Id: I3509892ec90b5a2135f7419f4c78209ac24e4e06
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39353
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Reviewed-by: AMIT KUMAR <akumar3@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

12 files changed, 597 insertions, 133 deletions

diff --git a/import/chips/p9/procedures/hwp/lib/p9_resclk_defines.H b/import/chips/p9/procedures/hwp/lib/p9_resclk_defines.H
index 9ba17158..25d83d86 100644
--- a/import/chips/p9/procedures/hwp/lib/p9_resclk_defines.H
+++ b/import/chips/p9/procedures/hwp/lib/p9_resclk_defines.H
@@ -57,12 +57,12 @@ std::vector<rsclk_freq_idx_t> const RESCLK_INDEX_VEC =
     //  { Freq, Idx}
     {    0,  3 },
     { 1500,  3 },
-    { 2000, 24 },
-    { 3000, 24 },
+    { 2000, 21 },
+    { 3000, 23 },
     { 3400, 24 },
-    { 3700, 24 },
-    { 3900, 24 },
-    { 4100, 24 }
+    { 3700, 22 },
+    { 3900, 20 },
+    { 4100, 19 }
 };
 //###############################################################################
 // Table 2: Resonant (Core & L2) Grids Control Data
@@ -149,15 +149,15 @@ std::vector<uint16_t> const RESCLK_TABLE_VEC =
 //###############################################################################
 std::vector<uint8_t> const L3CLK_TABLE_VEC
 {
-    0,
-    1,
+    2,
     3,
-    2
+    1,
+    0
 };
 //###############################################################################
 // L3 Voltage Threshold (millivolts)
 //###############################################################################
-uint16_t const L3_VOLTAGE_THRESHOLD_MV = 600;
+uint16_t const L3_VOLTAGE_THRESHOLD_MV = 580;
 }
 
 #endif //__P9_RESCLK_DEFINES_H__
diff --git a/import/chips/p9/procedures/ppe_closed/cme/cme_panic_codes.h b/import/chips/p9/procedures/ppe_closed/cme/cme_panic_codes.h
index c490d9db..dff06a63 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/cme_panic_codes.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/cme_panic_codes.h
@@ -76,10 +76,10 @@ CME_STOP_ENTRY_XSTOP_ERROR          = 0x1d09,
 //_UNUSED_1d1e                      = 0x1d1e,
 //_UNUSED_1d1f                      = 0x1d1f,
 
-//_UNUSED_1e00                      = 0x1e00, // Pstate Error
-//_UNUSED_1e01                      = 0x1e01,
-//_UNUSED_1e02                      = 0x1e02,
-//_UNUSED_1e03                      = 0x1e03,
+CME_PSTATE_RESCLK_ENABLED_AT_BOOT   = 0x1e00, // Pstate Error
+CME_PSTATE_IPPM_ACCESS_FAILED       = 0x1e01,
+CME_PSTATE_INVALID_DB0_MSGID        = 0x1e02,
+CME_PSTATE_UNEXPECTED_INTERCME_MSG  = 0x1e03,
 //_UNUSED_1e04                      = 0x1e04,
 //_UNUSED_1e05                      = 0x1e05,
 //_UNUSED_1e06                      = 0x1e06,
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
index b5aa77cd..9916071e 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
@@ -53,6 +53,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     /* 0: IDX_PRTY_VEC    1: IDX_MASK_VEC */
     {
         IRQ_VEC_PRTY0_CME, /* 0: IDX_PRTY_LVL_HIPRTY */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -68,6 +69,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY1_CME, /* 1: IDX_PRTY_LVL_DB3 */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -82,6 +84,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY2_CME, /* 2: IDX_PRTY_LVL_DB2 */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -95,6 +98,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY3_CME, /* 3: IDX_PRTY_LVL_SPWU */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -107,6 +111,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY4_CME, /* 4: IDX_PRTY_LVL_RGWU */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -118,6 +123,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY5_CME, /* 5: IDX_PRTY_LVL_PCWU */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -128,6 +134,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY6_CME, /* 6: IDX_PRTY_LVL_PM_ACTIVE */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -135,18 +142,18 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
         IRQ_VEC_PRTY7_CME |
         IRQ_VEC_PRTY6_CME
     },
-
     {
         IRQ_VEC_PRTY7_CME, /* 7: IDX_PRTY_LVL_DB1 */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
         IRQ_VEC_PRTY7_CME
     },
-
     {
         IRQ_VEC_PRTY8_CME, /* 8: IDX_PRTY_LVL_DB0 */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
@@ -154,18 +161,35 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
     },
     {
         IRQ_VEC_PRTY9_CME, /* 9: IDX_PRTY_LVL_INTERCME_IN0 */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME
     },
     {
         IRQ_VEC_PRTY11_CME, /* 10: IDX_PRTY_LVL_PMCR */
+        IRQ_VEC_PRTY12_CME |
         IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME
     },
     {
-        IRQ_VEC_PRTY11_CME, /* 11: IDX_PRTY_LVL_DISABLED */
-        IRQ_VEC_PRTY11_CME
+        IRQ_VEC_PRTY11_CME, /* 11: IDX_PRTY_LVL_COMM_RECVD */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
+        IRQ_VEC_PRTY10_CME |
+        IRQ_VEC_PRTY9_CME  |
+        IRQ_VEC_PRTY8_CME  |
+        IRQ_VEC_PRTY7_CME  |
+        IRQ_VEC_PRTY6_CME  |
+        IRQ_VEC_PRTY5_CME  |
+        IRQ_VEC_PRTY4_CME  |
+        IRQ_VEC_PRTY3_CME  |
+        IRQ_VEC_PRTY2_CME  |
+        IRQ_VEC_PRTY1_CME
+    },
+    {
+        IRQ_VEC_PRTY12_CME, /* 12: IDX_PRTY_LVL_DISABLED */
+        IRQ_VEC_PRTY12_CME
     }
 
 };
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
index f99653e8..2eef1960 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
@@ -58,10 +58,11 @@
 #define IDX_PRTY_LVL_DB0            8
 #define IDX_PRTY_LVL_INTERCME_IN0   9
 #define IDX_PRTY_LVL_PMCR           10
-#define IDX_PRTY_LVL_DISABLED       11
+#define IDX_PRTY_LVL_COMM_RECVD     11
+#define IDX_PRTY_LVL_DISABLED       12
 #define IDX_PRTY_VEC                0
 #define IDX_MASK_VEC                1
-#define NUM_EXT_IRQ_PRTY_LEVELS  (uint8_t)(12)
+#define NUM_EXT_IRQ_PRTY_LEVELS     13
 extern const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2];
 
 // Group0: Non-task hi-prty IRQs
@@ -86,8 +87,10 @@ extern const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2];
 #define IRQ_VEC_PRTY9_CME   (uint64_t)(0x0100000000000000)
 // Group10: PMCR
 #define IRQ_VEC_PRTY10_CME  (uint64_t)(0x0000000030000000)
-// Group11: We should never detect these
-#define IRQ_VEC_PRTY11_CME  (uint64_t)(0x00C003FFC33FFFFF)
+// Group11: COMM_RECVD
+#define IRQ_VEC_PRTY11_CME  (uint64_t)(0x0000000400000000)
+// Group12: We should never detect these
+#define IRQ_VEC_PRTY12_CME  (uint64_t)(0x00C003FBC33FFFFF)
 
 // This should be 0xFFFFFFFFFFFFFFFF
 #define IRQ_VEC_PRTY_CHECK  ( IRQ_VEC_PRTY0_CME | \
@@ -101,7 +104,8 @@ extern const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2];
                               IRQ_VEC_PRTY8_CME | \
                               IRQ_VEC_PRTY9_CME | \
                               IRQ_VEC_PRTY10_CME | \
-                              IRQ_VEC_PRTY11_CME )
+                              IRQ_VEC_PRTY11_CME | \
+                              IRQ_VEC_PRTY12_CME )
 
 extern uint8_t       g_current_prty_level;
 extern uint8_t       g_eimr_stack[NUM_EXT_IRQ_PRTY_LEVELS];
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_main.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_main.c
index fd03f2b2..240cab44 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_main.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_main.c
@@ -81,7 +81,7 @@ IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_CHTM_PURGE_DONE_C1
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_BCE_BUSY_LOW
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_FINAL_VDM_DATA0
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_FINAL_VDM_DATA1
-IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_COMM_RECVD
+IRQ_HANDLER(p9_cme_pstate_intercme_msg_handler, (void*)NULL) //CMEHW_IRQ_COMM_RECVD
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_COMM_SEND_ACK
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_COMM_SEND_NACK
 IRQ_HANDLER_DEFAULT                         //CMEHW_IRQ_SPARE_32
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
index a3ede88b..257c8736 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
@@ -36,6 +36,8 @@
 #include "pstate_pgpe_cme_api.h"
 //#include "p9_pstate_vpd.h"
 #include "ppe42_cache.h"
+#include "p9_cme_pstate.h"
+#include "cme_panic_codes.h"
 
 //
 //Globals
@@ -57,6 +59,8 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
 
     PK_TRACE("INTER0: Enter\n");
 
+    // TODO Revisit this loop as part of CME code review, PMSR update should
+    //      be done in one common function.
     for (c = 0; c < CORES_PER_EX; c++ )
     {
         if (cme_flags & (CME_FLAGS_CORE0_GOOD >> c))
@@ -74,6 +78,10 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
             //Determine PMSR
             localPS = (dbData.value >>
                        ((MAX_QUADS - G_cme_pstate_record.quadNum - 1) << 3)) & 0xFF;
+            // Update the Pstate variables
+            G_cme_pstate_record.quadPstate = localPS;
+            G_cme_pstate_record.globalPstate = (dbData.value & BITS64(8, 15)) >> SHIFT64(15);
+
             pmsrData = (dbData.value << 8) & 0xFF00000000000000;
             pmsrData |= ((uint64_t)localPS << 48) & 0x00FF000000000000;
             PK_TRACE_INF("INTER0:C%d PMSR=0x%08x%08x\n", c, pmsrData >> 32, pmsrData);
@@ -100,7 +108,7 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
             }
             else
             {
-                pk_halt();
+                PK_PANIC(CME_PSTATE_INVALID_DB0_MSGID);
             }
 
             out32_sh(CME_LCL_EISR_CLR, BIT32(4) >> c);//Clear DB0_C0/C1
@@ -115,3 +123,18 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
 
     PK_TRACE("INTER0: Exit\n");
 }
+
+void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq)
+{
+    PkMachineContext ctx;
+    // Override mask, disable every interrupt except high-priority ones via the
+    // priority mask for this interrupt (p9_pk_irq.c)
+    uint32_t msg;
+    intercme_msg_recv(&msg, IMT_LOCK_SIBLING);
+
+    // Block on the intercme0 interrupt
+    while(!(in32(CME_LCL_EISR) & BIT32(8))) {}
+
+    // Restore the mask, cede control to the intercme0 interrupt handler
+    pk_irq_vec_restore(&ctx);
+}
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
index f0cc54cb..f82e3ec9 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
@@ -50,13 +50,17 @@
 #include "cmehw_common.h"
 #include "cmehw_interrupts.h"
 #include "p9_cme_pstate.h"
+#include "p9_cme_stop.h" // For CmeStopRecord
 #include "p9_hcode_image_defines.H"
+#include "p9_cme_flags.h"
 
 //
 //Globals
 //
 cmeHeader_t* G_cmeHeader;
 LocalPstateParmBlock* G_lppb;
+extern CmePstateRecord G_cme_pstate_record;
+extern CmeStopRecord G_cme_stop_record;
 
 
 //
@@ -80,3 +84,262 @@ int send_pig_packet(uint64_t data, uint32_t coreMask)
 
     return rc;
 }
+
+void ippm_read(uint32_t addr, uint64_t* data)
+{
+    // G_cme_pstate_record.cmeMaskGoodCore MUST be set!
+    uint64_t val;
+
+    cppm_ippmcmd_t cppm_ippmcmd;
+    cppm_ippmcmd.value = 0;
+    cppm_ippmcmd.fields.qppm_reg = addr & 0x000000ff;
+    cppm_ippmcmd.fields.qppm_rnw = 1;
+    CME_PUTSCOM(CPPM_IPPMCMD, G_cme_pstate_record.cmeMaskGoodCore,
+                cppm_ippmcmd.value);
+
+    do
+    {
+        CME_GETSCOM(CPPM_IPPMSTAT, G_cme_pstate_record.cmeMaskGoodCore,
+                    CME_SCOM_EQ, val);
+    } // Check the QPPM_ONGOING bit
+
+    while(val & BIT64(0));
+
+    // QPPM_STATUS, non-zero indicates an error
+    if(val & BITS64(1, 2))
+    {
+        PK_PANIC(CME_PSTATE_IPPM_ACCESS_FAILED);
+    }
+
+    CME_GETSCOM(CPPM_IPPMRDATA, G_cme_pstate_record.cmeMaskGoodCore,
+                CME_SCOM_EQ, val);
+
+    *data = val;
+}
+
+void ippm_write(uint32_t addr, uint64_t data)
+{
+    // G_cme_pstate_record.cmeMaskGoodCore MUST be set!
+    uint64_t val;
+
+    CME_PUTSCOM(CPPM_IPPMWDATA, G_cme_pstate_record.cmeMaskGoodCore,
+                data);
+    cppm_ippmcmd_t cppm_ippmcmd;
+    cppm_ippmcmd.value = 0;
+    cppm_ippmcmd.fields.qppm_reg = addr & 0x000000ff;
+    cppm_ippmcmd.fields.qppm_rnw = 0;
+    CME_PUTSCOM(CPPM_IPPMCMD, G_cme_pstate_record.cmeMaskGoodCore,
+                cppm_ippmcmd.value);
+
+    do
+    {
+        CME_GETSCOM(CPPM_IPPMSTAT, G_cme_pstate_record.cmeMaskGoodCore,
+                    CME_SCOM_EQ, val);
+    } // Check the QPPM_ONGOING bit
+
+    while(val & BIT64(0));
+
+    // QPPM_STATUS, non-zero indicates an error
+    if(val & BITS64(1, 2))
+    {
+        PK_PANIC(CME_PSTATE_IPPM_ACCESS_FAILED);
+    }
+}
+
+void intercme_msg_send(uint32_t msg, INTERCME_MSG_TYPE type)
+{
+    out32(CME_LCL_ICSR, (msg << 4) | type);
+
+    PK_TRACE_DBG("imt send | msg=%08x", ((msg << 4) | type));
+    PK_TRACE_DBG("buff");
+
+    // Block on ack from companion CME
+    while(!(in32(CME_LCL_EISR) & BIT32(30))) {}
+
+    out32(CME_LCL_EISR_CLR, BIT32(30));
+}
+
+void intercme_msg_recv(uint32_t* msg, INTERCME_MSG_TYPE type)
+{
+    // Poll for inter-cme communication from QM
+    while(!(in32(CME_LCL_EISR) & BIT32(29))) {}
+
+    // Get the initial pstate value
+    *msg = in32(CME_LCL_ICRR);
+
+    PK_TRACE_DBG("imt recv | msg=%08x", *msg);
+
+    if(*msg & type)
+    {
+        // Shift out the type field, leaving only the message data
+        *msg >>= 4;
+    }
+    else
+    {
+        PK_PANIC(CME_PSTATE_UNEXPECTED_INTERCME_MSG);
+    }
+
+    // Ack back to companion CME that msg was received
+    out32(CME_LCL_ICCR_OR, BIT32(0));
+    // Clear the ack
+    out32(CME_LCL_ICCR_CLR, BIT32(0));
+    out32(CME_LCL_EISR_CLR, BIT32(29));
+}
+
+void p9_cme_resclk_get_index(uint32_t pstate, uint32_t* resclk_index)
+{
+    int32_t i = RESCLK_FREQ_REGIONS;
+
+    // Walk the table backwards by decrementing the index and checking for
+    // a value less than or equal to the requested pstate
+    // If no match is found then the index will be zero meaning resonance gets
+    // disabled
+    while((pstate > G_lppb->resclk.resclk_freq[--i]) && (i > 0)) {}
+
+    PK_TRACE_DBG("resclk_idx[i=%d]=%d", i, G_lppb->resclk.resclk_index[i]);
+    *resclk_index = (uint32_t)G_lppb->resclk.resclk_index[i];
+}
+
+void p9_cme_analog_control(uint32_t core_mask, ANALOG_CONTROL enable)
+{
+    if((in32(CME_LCL_FLAGS)) & CME_FLAGS_RCLK_OPERABLE)
+    {
+        uint32_t pstate;
+        uint32_t curr_idx;
+        uint64_t val;
+
+        if(enable)
+        {
+            PK_TRACE_INF("resclk | enabling resclks");
+
+            if(core_mask == CME_MASK_C0)
+            {
+                // Use Core0 index since only updating that core
+                curr_idx = G_cme_pstate_record.resclkData.core0_resclk_idx;
+            }
+            else
+            {
+                // Use Core1 index if a) only updating that core, or b) in the
+                // case of both Cores since the indices will be the same
+                curr_idx = G_cme_pstate_record.resclkData.core1_resclk_idx;
+            }
+
+            // 1) step CACCR to running pstate
+            pstate = G_cme_pstate_record.quadPstate;
+            p9_cme_resclk_update(core_mask, pstate, curr_idx);
+            // 2) write CACCR[13:15]=0b111 to switch back to common control
+            //    and leave clksync enabled
+            CME_PUTSCOM(CPPM_CACCR_OR, core_mask, (BITS64(13, 15)));
+
+            // Update PMSRS (only on stop-exit)
+            // TODO Revisit during CME code review, should use common PMSRS
+            //      function instead (for when other fields are added in the
+            //      future)
+            uint64_t pmsrs = ((((uint64_t)pstate << 48) & BITS64(8, 15))
+                              | (((uint64_t)G_cme_pstate_record.globalPstate << 56)
+                                 & BITS64(0, 7)));
+
+            if(core_mask & ANALOG_CORE0)
+            {
+                out64(CME_LCL_PMSRS0, pmsrs);
+            }
+
+            if(core_mask & ANALOG_CORE1)
+            {
+                out64(CME_LCL_PMSRS1, pmsrs);
+            }
+        }
+        else
+        {
+            PK_TRACE_INF("resclk | disabling resclks");
+
+            // 1) copy QACCR[0:12] into CACCR[0:12], with CACCR[13:15]=0b000,
+            //    to switch away from common control while leaving clksync
+            //    disabled. QACCR will already be set to a value corresponding
+            //    to the current quad Pstate
+            ippm_read(QPPM_QACCR, &val);
+            val &= BITS64(13, 63);
+            CME_PUTSCOM(CPPM_CACCR, core_mask, val);
+            p9_cme_resclk_get_index(G_cme_pstate_record.quadPstate, &curr_idx);
+            // 2) step CACCR to a value which disables resonance
+            pstate = ANALOG_PSTATE_RESCLK_OFF;
+            p9_cme_resclk_update(core_mask, pstate, curr_idx);
+        }
+    }
+}
+
+void p9_cme_resclk_update(ANALOG_TARGET target, uint32_t pstate, uint32_t curr_idx)
+{
+    uint64_t base_val;
+    uint64_t val;
+    uint32_t next_idx;
+    int32_t  step;
+
+    PK_TRACE_DBG("resclk | target=%08x", (uint32_t)target);
+    PK_TRACE_DBG("resclk | pstate=%d"  , pstate);
+
+    p9_cme_resclk_get_index(pstate, &next_idx);
+
+    PK_TRACE_DBG("resclk | curr_idx=%d", curr_idx);
+    PK_TRACE_DBG("resclk | next_idx=%d", next_idx);
+
+    // Determine the step polarity, step is not used if curr_idx == next_idx
+    if(curr_idx < next_idx)
+    {
+        step = 1;
+    }
+    else
+    {
+        step = -1;
+    }
+
+    // Read out the resclk register that is currently in control
+    if(target == ANALOG_COMMON)
+    {
+        ippm_read(QPPM_QACCR, &base_val);
+    }
+    else
+    {
+        CME_GETSCOM(CPPM_CACCR, target, CME_SCOM_EQ, base_val);
+    }
+
+    // Preserve only the resclk control bits
+    base_val &= (BITS64(13, 63));
+
+    while(curr_idx != next_idx)
+    {
+        curr_idx += step;
+        val = (((uint64_t)G_lppb->resclk.steparray[curr_idx].value) << 48)
+              | base_val;
+
+        if(target == ANALOG_COMMON)
+        {
+            ippm_write(QPPM_QACCR, val);
+        }
+        else
+        {
+            CME_PUTSCOM(CPPM_CACCR, target, val);
+        }
+
+        // There is an attribute for step-delay which is currently not used,
+        // this is where the delay would go.
+    }
+
+    // Update the resclk index variables
+    if(target == ANALOG_COMMON)
+    {
+        G_cme_pstate_record.resclkData.common_resclk_idx = curr_idx;
+    }
+    else
+    {
+        if(target & ANALOG_CORE0)
+        {
+            G_cme_pstate_record.resclkData.core0_resclk_idx = curr_idx;
+        }
+
+        if(target & ANALOG_CORE1)
+        {
+            G_cme_pstate_record.resclkData.core1_resclk_idx = curr_idx;
+        }
+    }
+}
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
index ec37e922..e43492d6 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
@@ -38,6 +38,7 @@
 #include "pk.h"
 #include "pstate_pgpe_cme_api.h"
 #include "gpehw_common.h"
+#include "cmehw_common.h"
 #include "p9_hcode_image_defines.H"
 #include "p9_pstates_cmeqm.h"
 
@@ -49,19 +50,55 @@ enum  PMCR_CONTROL
     PMCR1_ENABLE   =    0x8
 };
 
-void p9_cme_pstate_pmcr_thread(void*);
-void p9_cme_pstate_db_thread(void*);
-void p9_cme_pstate_pmcr_handler(void*, PkIrqId);
-void p9_cme_pstate_db_handler(void*, PkIrqId);
-void p9_cme_pstate_intercme_in0_handler(void*, PkIrqId);
-int send_pig_packet(uint64_t data, uint32_t coreMask);
+typedef struct
+{
+    uint32_t core0_resclk_idx;
+    uint32_t core1_resclk_idx;
+    uint32_t l2_ex0_resclk_idx;
+    uint32_t l2_ex1_resclk_idx;
+    uint32_t common_resclk_idx;
+} cme_resclk_data_t;
 
+typedef enum
+{
+    ANALOG_CORE0  = (uint32_t)CME_MASK_C0,
+    ANALOG_CORE1  = (uint32_t)CME_MASK_C1,
+    ANALOG_BOTHC  = (uint32_t)CME_MASK_BC,
+    ANALOG_L2EX0  = (uint32_t)0x00000100,
+    ANALOG_L2EX1  = (uint32_t)0x00000200,
+    ANALOG_BOTHL  = (uint32_t)0x00000300,
+    ANALOG_COMMON = (uint32_t)0x00001000
+} ANALOG_TARGET;
+
+typedef enum
+{
+    ANALOG_DISABLE = (uint32_t)0,
+    ANALOG_ENABLE  = (uint32_t)1
+} ANALOG_CONTROL;
+
+typedef enum
+{
+    ANALOG_PSTATE_RESCLK_OFF = (uint32_t)255
+} ANALOG_CONSTANTS;
+
+typedef enum
+{
+    IMT_INIT_PSTATE  = (uint32_t)0x00000001,
+    IMT_LOCK_SIBLING = (uint32_t)0x00000002,
+    IMT_SYNC_SIBLING = (uint32_t)0x00000003
+} INTERCME_MSG_TYPE;
 
 typedef struct
 {
     PkSemaphore sem[2];
+    uint32_t qmFlag;
+    uint32_t siblingCMEFlag;
     uint32_t quadNum;
     uint32_t pstatesEnabled;
+    uint32_t quadPstate;
+    uint32_t globalPstate;
+    uint32_t cmeMaskGoodCore;
+    cme_resclk_data_t resclkData;
 } CmePstateRecord;
 
 typedef struct
@@ -71,14 +108,25 @@ typedef struct
 
 typedef struct
 {
-    uint32_t qmFlag;
-    uint32_t siblingCMEFlag;
-    uint32_t cmeMaskGoodCore;
-    uint32_t globalPS;
-    uint32_t localPS;
-    uint32_t resClkTblIdx;
     uint32_t qaccr21_23InitVal;
     uint32_t dpll_pstate0_value;
 } cme_pstate_db_data_t;
 
+
+void p9_cme_pstate_pmcr_thread(void*);
+void p9_cme_pstate_db_thread(void*);
+void p9_cme_pstate_pmcr_handler(void*, PkIrqId);
+void p9_cme_pstate_db_handler(void*, PkIrqId);
+void p9_cme_pstate_intercme_in0_handler(void*, PkIrqId);
+void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq);
+int send_pig_packet(uint64_t data, uint32_t coreMask);
+void ippm_read(uint32_t addr, uint64_t* data);
+void ippm_write(uint32_t addr, uint64_t data);
+void intercme_msg_send(uint32_t msg, INTERCME_MSG_TYPE type);
+void intercme_msg_recv(uint32_t* msg, INTERCME_MSG_TYPE type);
+void p9_cme_resclk_get_index(uint32_t pstate, uint32_t* resclk_index);
+void p9_cme_analog_control(uint32_t core_mask, ANALOG_CONTROL enable);
+void p9_cme_resclk_update(ANALOG_TARGET target, uint32_t pstate, uint32_t curr_idx);
+
+
 #endif //_P9_CME_PSTATE_H_
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
index 0ed7f425..e7b245f4 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
@@ -46,6 +46,8 @@
 //#include "p9_pstate_vpd.h"
 #include "ppe42_cache.h"
 #include "p9_hcode_image_defines.H"
+#include "p9_cme_pstate.h"
+#include "cme_panic_codes.h"
 
 
 //
@@ -63,10 +65,10 @@ inline void p9_cme_pstate_process_db0();
 inline void p9_cme_pstate_db0_start(cppm_cmedb0_t dbData, uint32_t cme_flags);
 inline void p9_cme_pstate_db0_glb_bcast(cppm_cmedb0_t dbData, uint32_t cme_flags);
 inline void p9_cme_pstate_db0_suspend(cppm_cmedb0_t dbData, uint32_t cme_flags);
-inline void p9_cme_pstate_freq_update(uint64_t dbData);
-inline void p9_cme_pstate_resclk_update();
+inline void p9_cme_pstate_freq_update(uint32_t next_pstate);
 inline void p9_cme_pstate_pmsr_updt(uint64_t dbData, uint32_t cme_flags);
 inline void p9_cme_pstate_notify_sib();
+inline void p9_cme_pstate_update(uint64_t dbData, uint32_t cme_flags);
 
 //
 //Doorbell0 interrupt handler
@@ -88,6 +90,9 @@ void p9_cme_pstate_db_thread(void* arg)
     uint32_t cme_flags;
     PkMachineContext  ctx;
     uint32_t pir;
+    uint32_t cores = 0;
+    uint64_t scom_data;
+    uint32_t resclk_data;
 
     G_cmeHeader = (cmeHeader_t*)(CME_SRAM_HEADER_ADDR);
     G_lppb = (LocalPstateParmBlock*)(G_cmeHeader->g_cme_pstate_region_offset + CME_SRAM_BASE_ADDR);
@@ -114,29 +119,28 @@ void p9_cme_pstate_db_thread(void* arg)
         //CME0 is functional
         if (cme_flags & CME_FLAGS_SIBLING_FUNCTIONAL)
         {
-            G_db_thread_data.qmFlag = 0;
-            G_db_thread_data.siblingCMEFlag = 1;
-            //CME0 is not functional, CME1 is quadMgr
+            G_cme_pstate_record.qmFlag = 0;
+            G_cme_pstate_record.siblingCMEFlag = 1;
         }
         else
         {
-            G_db_thread_data.qmFlag = 1;
-            G_db_thread_data.siblingCMEFlag = 0;
+            //CME0 is not functional, so CME1 is quadMgr
+            G_cme_pstate_record.qmFlag = 1;
+            G_cme_pstate_record.siblingCMEFlag = 0;
         }
     }
-    //CME 0 is always the quad manager
     else
     {
-        //CME 0 is always the quad manager
-        G_db_thread_data.qmFlag = 1;
+        // A functional CME0 is always the quad manager
+        G_cme_pstate_record.qmFlag = 1;
 
         if (cme_flags & CME_FLAGS_SIBLING_FUNCTIONAL)
         {
-            G_db_thread_data.siblingCMEFlag = 1;
+            G_cme_pstate_record.siblingCMEFlag = 1;
         }
         else
         {
-            G_db_thread_data.siblingCMEFlag = 0;
+            G_cme_pstate_record.siblingCMEFlag = 0;
         }
     }
 
@@ -146,40 +150,134 @@ void p9_cme_pstate_db_thread(void* arg)
     //siblingCME.
     //
     //if quadManager
-    if (G_db_thread_data.qmFlag)
+    if (G_cme_pstate_record.qmFlag)
     {
         if (cme_flags & CME_FLAGS_CORE0_GOOD)
         {
             out32_sh(CME_LCL_EIMR_CLR, BIT32(4));//Enable DB0_0
             out32_sh(CME_LCL_EIMR_OR, BIT32(5));//Disable DB0_1
             g_eimr_override |= BIT64(37);
-            G_db_thread_data.cmeMaskGoodCore = CME_MASK_C0;
+            G_cme_pstate_record.cmeMaskGoodCore = CME_MASK_C0;
+            cores |= CME_MASK_C0;
         }
         else if (cme_flags & CME_FLAGS_CORE1_GOOD)
         {
             out32_sh(CME_LCL_EIMR_OR, BIT32(4));//Disable DB0_0
             out32_sh(CME_LCL_EIMR_CLR, BIT32(5));//Enable DB0_1
             g_eimr_override |= BIT64(36);
-            G_db_thread_data.cmeMaskGoodCore = CME_MASK_C1;
+            G_cme_pstate_record.cmeMaskGoodCore = CME_MASK_C1;
+            cores |= CME_MASK_C1;
         }
 
         out64(CME_LCL_EIMR_OR, BIT64(7));//Disable  InterCME_IN0
         g_eimr_override |= BIT64(7);
         G_db_thread_data.dpll_pstate0_value   = G_lppb->dpll_pstate0_value;
+
+        // Pstate Clocking Initialization (QM)
+        // Calculate the initial pstate
+        ippm_read(QPPM_DPLL_STAT, &scom_data);
+        G_cme_pstate_record.quadPstate = (uint32_t)G_db_thread_data.dpll_pstate0_value
+                                         - (uint32_t)((scom_data & BITS64(1, 11)) >> SHIFT64(11));
+        PK_TRACE_INF("qm | initial pstate=%d", G_cme_pstate_record.quadPstate);
+
+        // Synchronize initial pstate w/ sibling CME
+        if(G_cme_pstate_record.siblingCMEFlag)
+        {
+            out64(CME_LCL_EITR_OR, BIT64(30));
+            out64(CME_LCL_EIPR_OR, BIT64(30));
+            intercme_msg_send(G_cme_pstate_record.quadPstate,
+                              IMT_INIT_PSTATE);
+        }
     }
     else
     {
         out64(CME_LCL_EIMR_OR, BIT64(36) | BIT64(37));//Disable DB0_0 and DB0_1
+
+        if(cme_flags & CME_FLAGS_CORE0_GOOD)
+        {
+            G_cme_pstate_record.cmeMaskGoodCore = CME_MASK_C0;
+            cores |= CME_MASK_C0;
+        }
+        else if(cme_flags & CME_FLAGS_CORE1_GOOD)
+        {
+            G_cme_pstate_record.cmeMaskGoodCore = CME_MASK_C1;
+            cores |= CME_MASK_C1;
+        }
+
+        // Resonant Clocking Initialization (Sibling)
+        out64(CME_LCL_EITR_OR, BIT64(29));
+        out64(CME_LCL_EIPR_OR, BIT64(29));
+
+        intercme_msg_recv(&G_cme_pstate_record.quadPstate, IMT_INIT_PSTATE);
+        PK_TRACE_INF("sib | initial pstate=%d", G_cme_pstate_record.quadPstate);
+
         out64(CME_LCL_EIMR_CLR, BIT64(7)); //Enable InterCME_IN0
         g_eimr_override |= BIT64(37);
         g_eimr_override |= BIT64(36);
+    }
+
+    // At this point, both QM and Sib can set their initial global Pstate
+    G_cme_pstate_record.globalPstate = G_cme_pstate_record.quadPstate;
+
+    // Resonant Clocking Check (QM + Sibling)
+    // Check that resonance is not enabled in CACCR and EXCGCR
+    CME_GETSCOM(CPPM_CACCR, cores, CME_SCOM_EQ, scom_data);
+    // Ignore clk_sync_enable and reserved
+    resclk_data = (scom_data >> 32) & ~BITS32(15, 31);
+
+    if(resclk_data != 0)
+    {
+        PK_PANIC(CME_PSTATE_RESCLK_ENABLED_AT_BOOT);
+    }
+
+    ippm_read(QPPM_EXCGCR, &scom_data);
+    // Ignore clk_sync_enable, clkglm_async_reset, clkglm_sel, and reserved
+    scom_data &= ~(BITS64(29, 37) | BITS64(42, 63));
 
+    if(scom_data != 0)
+    {
+        PK_PANIC(CME_PSTATE_RESCLK_ENABLED_AT_BOOT);
+    }
+
+    // Resonant Clocking Initialization (QM + Sibling)
+    if(G_cmeHeader->g_cme_qm_mode_flags & CME_QM_FLAG_RESCLK_ENABLE)
+    {
+        // Initialize the Resclk indices
+        G_cme_pstate_record.resclkData.core0_resclk_idx =
+            (uint32_t)G_lppb->resclk.resclk_index[0];
+        G_cme_pstate_record.resclkData.core1_resclk_idx =
+            (uint32_t)G_lppb->resclk.resclk_index[0];
+
+        if(G_cme_pstate_record.qmFlag)
+        {
+            G_cme_pstate_record.resclkData.l2_ex0_resclk_idx =
+                (uint32_t)G_lppb->resclk.resclk_index[0];
+            G_cme_pstate_record.resclkData.l2_ex1_resclk_idx =
+                (uint32_t)G_lppb->resclk.resclk_index[0];
+
+            // Extract the resclk value from QCCR
+            ippm_read(QPPM_QACCR, &scom_data);
+            scom_data = (scom_data & BITS64(0, 12)) >> SHIFT64(15);
+            p9_cme_resclk_get_index(G_cme_pstate_record.quadPstate,
+                                    &G_cme_pstate_record.resclkData.common_resclk_idx);
+            // Read QACCR and clear out the resclk settings
+            ippm_read(QPPM_QACCR, &scom_data);
+            scom_data &= ~BITS64(0, 12);
+            // OR-in the resclk settings which match the current Pstate
+            scom_data |= (((uint64_t)G_lppb->resclk.steparray
+                           [G_cme_pstate_record.resclkData.common_resclk_idx].value)
+                          << 48);
+            // Write QACCR
+            ippm_write(QPPM_QACCR, scom_data);
+        }
+
+        out32(CME_LCL_FLAGS_OR, CME_FLAGS_RCLK_OPERABLE);
     }
 
     //Doorbell Thread(this thread) will continue to run on
     //Quad Manager CME. The sibling CME has intercme_in0 enabled
     //and won't run this thread past this point.
-    if (G_db_thread_data.qmFlag)
+    if (G_cme_pstate_record.qmFlag)
     {
         pk_semaphore_create(&G_cme_pstate_record.sem[1], 0, 1);
 
@@ -261,7 +359,7 @@ inline void p9_cme_pstate_db0_start(cppm_cmedb0_t dbData, uint32_t cme_flags)
         ppmPigData.value = 0;
         ppmPigData.fields.req_intr_type = 4;
         ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_ERROR;
-        send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+        send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
         PK_TRACE_INF("DB_TH: DB0  Start while already started\n");
         pk_halt();
     }
@@ -272,13 +370,7 @@ inline void p9_cme_pstate_db0_start(cppm_cmedb0_t dbData, uint32_t cme_flags)
     //Check for iVRM disable
     //Check for VDM disable
 
-    //Pstate Update
-#if !SIMICS_TUNING
-    p9_cme_pstate_freq_update(dbData.value);
-    p9_cme_pstate_notify_sib(); //Notify sibling
-#endif
-
-    p9_cme_pstate_pmsr_updt(dbData.value, cme_flags);
+    p9_cme_pstate_update(dbData.value, cme_flags);
 
     //\TODO RTC: 152965
     //Enable Resonant Clks if flag
@@ -291,7 +383,7 @@ inline void p9_cme_pstate_db0_start(cppm_cmedb0_t dbData, uint32_t cme_flags)
     ppmPigData.value = 0;
     ppmPigData.fields.req_intr_type = 4;
     ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK;
-    send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+    send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
 
     //Clear Pending PMCR interrupts and Enable PMCR Interrupts
     if (cme_flags & CME_FLAGS_CORE0_GOOD)
@@ -323,30 +415,18 @@ inline void p9_cme_pstate_db0_glb_bcast(cppm_cmedb0_t dbData, uint32_t cme_flags
         ppmPigData.value = 0;
         ppmPigData.fields.req_intr_type = 4;
         ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_ERROR;
-        send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+        send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
         PK_TRACE_INF("DB_TH: DB0Bcast while PS disabled\n");
         pk_halt();
     }
 
-    //Update analog
-    /*
-    if (G_resclkEnabled)
-    {
-        p9_cme_pstate_resclk_update();
-    }
-    */
-
-#if !SIMICS_TUNING
-    p9_cme_pstate_freq_update(dbData.value);
-    p9_cme_pstate_notify_sib(); //Notify sibling
-#endif
-    p9_cme_pstate_pmsr_updt(dbData.value, cme_flags);
+    p9_cme_pstate_update(dbData.value, cme_flags);
 
     //Send type4(ack doorbell)
     ppmPigData.value = 0;
     ppmPigData.fields.req_intr_type = 4;
     ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK;
-    send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+    send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
     PK_TRACE_INF("DB_TH: DB0 GlbBcast Exit\n");
 }
 
@@ -364,7 +444,7 @@ inline void p9_cme_pstate_db0_suspend(cppm_cmedb0_t dbData, uint32_t cme_flags)
         ppmPigData.value = 0;
         ppmPigData.fields.req_intr_type = 4;
         ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_ERROR;
-        send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+        send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
         pk_halt();
     }
 
@@ -390,7 +470,7 @@ inline void p9_cme_pstate_db0_suspend(cppm_cmedb0_t dbData, uint32_t cme_flags)
     ppmPigData.value = 0;
     ppmPigData.fields.req_intr_type = 4;
     ppmPigData.fields.req_intr_payload = MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED;
-    send_pig_packet(ppmPigData.value, G_db_thread_data.cmeMaskGoodCore);
+    send_pig_packet(ppmPigData.value, G_cme_pstate_record.cmeMaskGoodCore);
     PK_TRACE_INF("DB_TH: DB0 Suspend Exit\n");
 }
 
@@ -432,7 +512,7 @@ inline void p9_cme_pstate_notify_sib()
     PK_TRACE_INF("DB_TH: Notify Enter\n");
 
     //Notify sibling CME(if any)
-    if (G_db_thread_data.siblingCMEFlag == 1)
+    if (G_cme_pstate_record.siblingCMEFlag == 1)
     {
         //Send interCME interrupt
         out32(CME_LCL_ICCR_OR, BIT32(5)); //Send direct InterCME_IN0
@@ -464,81 +544,76 @@ inline void p9_cme_pstate_notify_sib()
 //
 //p9_cme_pstate_freq_update
 //
-inline void p9_cme_pstate_freq_update(uint64_t dbData)
+inline void p9_cme_pstate_freq_update(uint32_t next_pstate)
 {
     PK_TRACE_INF("DB_TH: Freq Updt Enter\n");
-    uint8_t localPS = (dbData >>
-                       ((MAX_QUADS - G_cme_pstate_record.quadNum - 1) << 3)) & 0xFF;
-
-    PK_TRACE_INF("DB_TH: DBData=0x%08x%08x\n", dbData >> 32, dbData);
     PK_TRACE_INF("DB_TH: Dpll0=0x%x\n", G_db_thread_data.dpll_pstate0_value);
     PK_TRACE_INF("DB_TH: Hdr=0x%x, LPPB=0x%x\n", (uint32_t)G_cmeHeader, (uint32_t)G_lppb);
     //Adjust DPLL
-
-    cppm_ippmcmd_t  cppm_ippmcmd;
     qppm_dpll_freq_t dpllFreq;
 
     //Write new value of DPLL using INTERPPM
     dpllFreq.value = 0;
-    dpllFreq.fields.fmax  = (uint16_t)(G_db_thread_data.dpll_pstate0_value - localPS);
-    dpllFreq.fields.fmult = (uint16_t)(G_db_thread_data.dpll_pstate0_value - localPS);
-    dpllFreq.fields.fmin  = (uint16_t)(G_db_thread_data.dpll_pstate0_value - localPS);
-    CME_PUTSCOM(CPPM_IPPMWDATA, G_db_thread_data.cmeMaskGoodCore, dpllFreq.value);
-    cppm_ippmcmd.value = 0;
-    cppm_ippmcmd.fields.qppm_reg = QPPM_DPLL_FREQ & 0x000000ff;
-    cppm_ippmcmd.fields.qppm_rnw = 0;
-    CME_PUTSCOM(CPPM_IPPMCMD, G_db_thread_data.cmeMaskGoodCore, cppm_ippmcmd.value);
+    dpllFreq.fields.fmax  = (uint16_t)(G_db_thread_data.dpll_pstate0_value - next_pstate);
+    dpllFreq.fields.fmult = (uint16_t)(G_db_thread_data.dpll_pstate0_value - next_pstate);
+    dpllFreq.fields.fmin  = (uint16_t)(G_db_thread_data.dpll_pstate0_value - next_pstate);
+    ippm_write(QPPM_DPLL_FREQ, dpllFreq.value);
     PK_TRACE_INF("DB_TH: Freq Updt Exit\n");
 }
 
-//
-//p9_cme_pstate_resclk_update
-//
-/*
-inline void p9_cme_pstate_resclk_update()
+inline void p9_cme_pstate_update(uint64_t dbData, uint32_t cme_flags)
 {
-    uint64_t val;
-    uint8_t tidx, step;
-    int32_t i;
+    PK_TRACE_INF("DB_TH: Pstate Updt Enter");
 
-    //get targetIndex from Table1(ControlIndex) by indexing with localPState
-    tidx = G_db_thread_data.resClkTblIdx;
-
-    for (i = NUM_FREQ_REGIONS - 1; i >= 0; i--)
+    if(G_cme_pstate_record.siblingCMEFlag)
     {
-        if (G_freq2idx[i].pstate > G_db_thread_data.localPS)
-        {
-            tidx = i;
-            break;
-        }
+        // "Lock" the sibling until the pstate transition is complete
+        intercme_msg_send(0, IMT_LOCK_SIBLING);
+        // The Sibling is a "pumpkin" from this point forward until calling
+        // p9_cme_pstate_notify_sib()
     }
 
-    //walk Table2[Resonant Grids Control Data)
-    if (tidx > G_db_thread_data.resClkTblIdx)
+    uint32_t next_pstate = (dbData >>
+                            ((MAX_QUADS - G_cme_pstate_record.quadNum - 1) << 3)) & 0x000000ff;
+    G_cme_pstate_record.globalPstate = (dbData & BITS64(8, 15)) >> SHIFT64(15);
+
+    PK_TRACE_INF("DB_TH: DBData=0x%08x%08x\n", dbData >> 32, dbData);
+
+    if(next_pstate > G_cme_pstate_record.quadPstate)
     {
-        step = 1;
+        p9_cme_pstate_freq_update(next_pstate);
+
+        if(cme_flags & CME_FLAGS_RCLK_OPERABLE)
+        {
+            PkMachineContext ctx;
+            pk_critical_section_enter(&ctx);
+
+            p9_cme_resclk_update(ANALOG_COMMON, next_pstate,
+                                 G_cme_pstate_record.resclkData.common_resclk_idx);
+
+            pk_critical_section_exit(&ctx);
+        }
     }
-    else
+    else if(next_pstate < G_cme_pstate_record.quadPstate)
     {
-        step = -1;
-    }
+        if(cme_flags & CME_FLAGS_RCLK_OPERABLE)
+        {
+            PkMachineContext ctx;
+            pk_critical_section_enter(&ctx);
 
-    while(G_db_thread_data.resClkTblIdx != tidx)
-    {
-        G_db_thread_data.resClkTblIdx += step;
-        val = (uint64_t)(G_cgm_table[G_db_thread_data.resClkTblIdx]) << 48;
-        val |= G_db_thread_data.qaccr21_23InitVal;
+            p9_cme_resclk_update(ANALOG_COMMON, next_pstate,
+                                 G_cme_pstate_record.resclkData.common_resclk_idx);
 
-#if !SIMICS_TUNING
-        cppm_ippmcmd_t  cppm_ippmcmd;
-        //Write val to QACCR
-        CME_PUTSCOM(CPPM_IPPMWDATA, G_db_thread_data.cmeMaskGoodCore, val);
-        cppm_ippmcmd.value = 0;
-        cppm_ippmcmd.fields.qppm_reg = QPPM_QACCR & 0x000000ff;
-        cppm_ippmcmd.fields.qppm_rnw = 0;
-        CME_PUTSCOM(CPPM_IPPMCMD, G_db_thread_data.cmeMaskGoodCore, cppm_ippmcmd.value);
-#endif
+            pk_critical_section_exit(&ctx);
+        }
+
+        p9_cme_pstate_freq_update(next_pstate);
     }
-}
 
-*/
+    p9_cme_pstate_notify_sib(); //Notify sibling
+    p9_cme_pstate_pmsr_updt(dbData, cme_flags);
+
+    G_cme_pstate_record.quadPstate = next_pstate;
+
+    PK_TRACE_INF("DB_TH: Pstate Updt Exit");
+}
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
index 61b807a9..473f1abc 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
@@ -45,6 +45,7 @@
 #include "pstate_pgpe_cme_api.h"
 //#include "p9_pstate_vpd.h"
 #include "ppe42_cache.h"
+#include "cme_panic_codes.h"
 
 //
 //Globals
@@ -73,6 +74,7 @@ void p9_cme_pstate_pmcr_thread(void* arg)
     uint32_t eisr;
     uint32_t coreMask[CORES_PER_EX];
     uint32_t cme_flags;
+    uint32_t msg;
     coreMask[0] = CME_MASK_C0;
     coreMask[1] = CME_MASK_C1;
 
@@ -82,6 +84,25 @@ void p9_cme_pstate_pmcr_thread(void* arg)
 
     pk_semaphore_create(&G_cme_pstate_record.sem[0], 0, 1);
 
+    // Synchronization between QM and Sibling
+    // @todo RTC173279 move into CME init function
+    if(G_cme_pstate_record.qmFlag)
+    {
+        // Synchronize QACCR setting w/ sibling CME
+        if(G_cme_pstate_record.siblingCMEFlag)
+        {
+            intercme_msg_send(0, IMT_SYNC_SIBLING);
+        }
+    }
+    else
+    {
+        intercme_msg_recv(&msg, IMT_SYNC_SIBLING);
+        // Unmask the COMM_RECVD interrupt for the intercme msg handler
+        out64(CME_LCL_EIMR_CLR, BIT64(29));
+    }
+
+    // This is the current barrier for SGPE booting the CMEs, any and all
+    // initialization must be completed prior!
     out32(CME_LCL_FLAGS_OR, CME_FLAGS_PMCR_READY);
 
     PK_TRACE_INF("PMCR_TH: Inited\n");
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
index 87a60644..72fda6a8 100755
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -45,6 +45,7 @@
 
 #include "p9_cme_stop.h"
 #include "p9_cme_stop_enter_marks.h"
+#include "p9_cme_pstate.h"
 
 extern CmeStopRecord G_cme_stop_record;
 
@@ -710,6 +711,8 @@ p9_cme_stop_entry()
         }
         while(scom_data.words.upper & BIT32(13));
 
+        p9_cme_analog_control(core, ANALOG_DISABLE);
+
         PK_TRACE("Switch glsmux to refclk to save clock grid power via CGCR[3]");
         CME_PUTSCOM(C_PPM_CGCR, core, 0);
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
index 230480a9..55615c8c 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
@@ -26,6 +26,7 @@
 #include "p9_cme_stop.h"
 #include "p9_cme_stop_exit_marks.h"
 #include "p9_cme_copy_scan_ring.h"
+#include "p9_cme_pstate.h"
 
 extern CmeStopRecord G_cme_stop_record;
 
@@ -286,6 +287,8 @@ void p9_cme_stop_exit_lv2(uint32_t core)
     PK_TRACE("Assert core glitchless mux to DPLL via CGCR[3]");
     CME_PUTSCOM(C_PPM_CGCR, core, BIT64(3));
 
+    p9_cme_analog_control(core, ANALOG_ENABLE);
+
     // do this after assert glsmux so glitch can have time to resolve
     // catchup to stop2 exit will acquire here