PM: VDM Prolonged Droop Fix

    Key_Cronus_Test=PM_REGRESS

Change-Id: I842b6100bf8537425fe1f0ac92bea44f75051348
Original-Change-Id: I73d38d6029a5b84590d1081855e12c145a535869
CQ: SW413192
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/51338
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Tested-by: Cronus HW CI <cronushw-ci+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: RANGANATHPRASAD G. BRAHMASAMUDRA <prasadbgr@in.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

7 files changed, 609 insertions, 184 deletions

diff --git a/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h b/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
index 5492995a..b0785f57 100644
--- a/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
+++ b/import/chips/p9/common/pmlib/include/pstate_pgpe_cme_api.h
@@ -98,24 +98,44 @@ enum MESSAGE_ID_DB0
     MSGID_DB0_START_PSTATE_BROADCAST    = 3,
     MSGID_DB0_STOP_PSTATE_BROADCAST     = 4,
     MSGID_DB0_CLIP_BROADCAST            = 5,
-    MSGID_DB0_SAFE_MODE_BROADCAST       = 6,
+    MSGID_DB0_PMSR_UPDT                 = 6,
     MSGID_DB0_VALID_END                 = 6 //This for error checking
 };
 
+enum MESSAGE_ID_DB3
+{
+    MSGID_DB3_RESERVED                  = 0,
+    MSGID_DB3_VALID_START               = 1, //This for error checking
+    MSGID_DB3_PSTATE_START              = 1,
+    MSGID_DB3_PSTATE_END                = 0xF0,
+    MSGID_DB3_IMMEDIATE_HALT            = 0xF1,
+    MSGID_DB3_RESTORE_STATE_AND_HALT    = 0xF2,
+    MSGID_DB3_REPLAY_DB0                = 0xF3,
+    MSGID_DB3_VALID_END                 = 0xF3 //This for error checking
+};
+
 enum MESSAGEID_PCB_TYPE4_ACK_TYPES
 {
     MSGID_PCB_TYPE4_ACK_ERROR                    = 0,
     MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK         = 1,
     MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED         = 2,
-    MSGID_PCB_TYPE4_QUAD_MGR_AVAILABLE           = 3
+    MSGID_PCB_TYPE4_QUAD_MGR_AVAILABLE           = 3,
+    MSGID_PCB_TYPE4_NACK_DROOP_PRESENT           = 4
 };
 
-enum DB0_FIELDS
+enum DB0_CLIP_BCAST_FIELDS
 {
     DB0_CLIP_BCAST_TYPE_PMIN      = 0,
     DB0_CLIP_BCAST_TYPE_PMAX      = 1
 };
 
+enum DB0_PMSR_UPDT_COMMANDS
+{
+    DB0_PMSR_UPDT_SET_SAFE_MODE             = 0x0,
+    DB0_PMSR_UPDT_SET_PSTATES_SUSPENDED     = 0x1,
+    DB0_PMSR_UPDT_CLEAR_PSTATES_SUSPENDED   = 0x2
+};
+
 //
 //PGPE-CME Doorbell0(Global Actual Broadcast)
 //
@@ -225,22 +245,24 @@ typedef union pgpe_db0_clip_bcast
 } pgpe_db0_clip_bcast_t;
 
 //
-//PGPE-CME Doorbell0(Safe Mode Broadcast)
+//PGPE-CME Doorbell0(PMSR Update)
 //
-typedef union pgpe_db0_safe_mode_bcast
+typedef union pgpe_db0_pmsr_updt
 {
     uint64_t value;
     struct
     {
 #ifdef _BIG_ENDIAN
         uint64_t msg_id : 8;
-        uint64_t reserved: 56;
+        uint64_t command : 8;
+        uint64_t reserved: 48;
 #else
-        uint64_t reserved: 56;
+        uint64_t reserved: 48;
+        uint64_t command : 8;
         uint64_t msg_id : 8;
 #endif
     } fields;
-} pgpe_db0_safe_mode_bcast_t;
+} pgpe_db0_pmsr_updt_t;
 
 
 #endif //__PSTATE_PGPE_CME_API_H__
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
index c4c3d7d7..958ca201 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
@@ -49,33 +49,89 @@ extern CmeRecord G_cme_record;
 //
 void p9_cme_pstate_intercme_in0_irq_handler(void* arg, PkIrqId irq)
 {
-    p9_cme_pstate_intercme_in0_handler();
+    PkMachineContext  ctx __attribute__((unused));
+
+    //Read DB0 from first good core since PGPE
+    //writes same value for both cores
+    uint64_t dbData;
+
+    if (in32(CME_LCL_FLAGS) & (BIT32(CME_FLAGS_CORE0_GOOD)))
+    {
+        CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C0, dbData);
+    }
+    else
+    {
+        CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C1, dbData);
+    }
+
+    p9_cme_pstate_process_db0_sibling(dbData);
+
+    pk_irq_vec_restore(&ctx);
 }
 
-void p9_cme_pstate_intercme_in0_handler()
+void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq)
 {
-    cppm_cmedb0_t dbData;
-    dbData.value = 0;
-    uint32_t dbQuadInfo, dbBit8_15;
-    uint32_t cme_flags = in32(CME_LCL_FLAGS);
     PkMachineContext  ctx __attribute__((unused));
 
-    PK_TRACE("INTER0: Enter\n");
+    p9_cme_pstate_sibling_lock_and_intercme_protocol(0, 1, 0);
 
-    //Read DB0 from first good core since PGPE
-    //writes same value for both cores
-    do
+    pk_irq_vec_restore(&ctx);
+}
+
+void p9_cme_pstate_sibling_lock_and_intercme_protocol(uint32_t process_intercme_in0, uint32_t readDB0, uint64_t dbData)
+{
+    PK_TRACE("SIBL: Enter\n");
+    uint32_t msg;
+    uint64_t data;
+    intercme_msg_recv(&msg, IMT_LOCK_SIBLING);
+
+    // Block on the intercme0/intercme1 interrupt
+    while((!(in32(CME_LCL_EISR) & BIT32(7))) &&
+          (!(in32_sh(CME_LCL_EISR) & BIT64SH(38)))) {}
+
+    //If INTERCME_DIRECT_IN1, then error.
+    if(in32_sh(CME_LCL_EISR) & BIT64SH(38))
     {
-        if (cme_flags & (BIT32(CME_FLAGS_CORE0_GOOD)))
+        G_cme_pstate_record.pstatesSuspended = 1;
+        p9_cme_pstate_pmsr_updt(G_cme_record.core_enabled);
+        intercme_direct(INTERCME_DIRECT_IN1, INTERCME_DIRECT_ACK, 0);
+    }
+
+    //If INTERCME_DIRECT_IN0, then process DB0 data
+    if((in32(CME_LCL_EISR) & BIT32(7)) && (process_intercme_in0 == 1))
+    {
+        if(readDB0)
         {
-            CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C0, dbData.value);
+            //Read DB0 from first good core since PGPE
+            //writes same value for both cores
+            if (in32(CME_LCL_FLAGS) & (BIT32(CME_FLAGS_CORE0_GOOD)))
+            {
+                CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C0, data);
+            }
+            else
+            {
+                CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C1, data);
+            }
         }
         else
         {
-            CME_GETSCOM(CPPM_CMEDB0, CME_MASK_C1, dbData.value);
+            data = dbData;
         }
+
+        p9_cme_pstate_process_db0_sibling(data);
     }
-    while(dbData.value == 0);
+
+    PK_TRACE("SIBL: Enter\n");
+}
+
+void p9_cme_pstate_process_db0_sibling(uint64_t data)
+{
+    cppm_cmedb0_t dbData;
+    dbData.value = data;
+    uint32_t dbQuadInfo, dbBit8_15;
+    //uint32_t cme_flags = in32(CME_LCL_FLAGS);
+
+    PK_TRACE("INTER0: Enter\n");
 
     dbQuadInfo = (dbData.value >> (in32(CME_LCL_SRTCH0) &
                                    (BITS32(CME_SCRATCH_LOCAL_PSTATE_IDX_START, CME_SCRATCH_LOCAL_PSTATE_IDX_LENGTH)
@@ -119,7 +175,6 @@ void p9_cme_pstate_intercme_in0_handler()
         }
 
         p9_cme_pstate_pmsr_updt(G_cme_record.core_enabled);
-        PKTRACE("INTER0: pmin=0x%08x,pmax=0x%08x", G_cme_pstate_record.pmin, G_cme_pstate_record.pmax);
     }
     else if(dbData.fields.cme_message_number0 == MSGID_DB0_STOP_PSTATE_BROADCAST)
     {
@@ -134,11 +189,24 @@ void p9_cme_pstate_intercme_in0_handler()
         //Set Core GPMMR RESET_STATE_INDICATOR bit to show pstates have stopped
         CME_PUTSCOM(PPM_GPMMR_OR, G_cme_record.core_enabled, BIT64(15));
     }
-    else if(dbData.fields.cme_message_number0 == MSGID_DB0_SAFE_MODE_BROADCAST)
+    else if(dbData.fields.cme_message_number0 == MSGID_DB0_PMSR_UPDT)
     {
-        PK_TRACE("INTER0: DB0 Safe Mode");
+        PK_TRACE("INTER0: DB0 PMSR Updt");
+
+        switch(dbBit8_15)
+        {
+            case DB0_PMSR_UPDT_SET_SAFE_MODE:
+                G_cme_pstate_record.safeMode = 1;
+                break;
 
-        G_cme_pstate_record.safeMode = 1;
+            case DB0_PMSR_UPDT_SET_PSTATES_SUSPENDED:
+                G_cme_pstate_record.pstatesSuspended = 1;
+                break;
+
+            case DB0_PMSR_UPDT_CLEAR_PSTATES_SUSPENDED:
+                G_cme_pstate_record.pstatesSuspended = 0;
+                break;
+        }
 
         p9_cme_pstate_pmsr_updt(G_cme_record.core_enabled);
     }
@@ -151,22 +219,5 @@ void p9_cme_pstate_intercme_in0_handler()
 
     intercme_direct(INTERCME_DIRECT_IN0, INTERCME_DIRECT_ACK, 0);
 
-    pk_irq_vec_restore(&ctx);
-
     PK_TRACE("INTER0: Exit");
 }
-
-void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq)
-{
-    PkMachineContext ctx __attribute__((unused));
-    // Override mask, disable every interrupt except high-priority ones via the
-    // priority mask for this interrupt (p9_pk_irq.c)
-    uint32_t msg;
-    intercme_msg_recv(&msg, IMT_LOCK_SIBLING);
-
-    // Block on the intercme0 interrupt
-    while(!(in32(CME_LCL_EISR) & BIT32(7))) {}
-
-    // Restore the mask, cede control to the intercme0 interrupt handler
-    pk_irq_vec_restore(&ctx);
-}
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
index b5bf3379..b87635f3 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
@@ -88,19 +88,87 @@ int send_pig_packet(uint64_t data, uint32_t coreMask)
     return rc;
 }
 
-void poll_dpll_update_complete()
+uint32_t poll_dpll_stat()
 {
-    data64_t polldata;
-    PK_TRACE_INF("Poll on DPLL_STAT[update_complete]");
+    data64_t data;
+    uint32_t cme_flags = in32(CME_LCL_FLAGS);
+    uint32_t rc = 0;
 
-    // ... to indicate that the DPLL has sampled the newly requested
-    // frequency into its internal registers as a target,
-    // but may not yet be there
-    do
+    // DPLL Mode 2
+    if(!(cme_flags & BIT32(CME_FLAGS_VDM_OPERABLE)))
     {
-        ippm_read(QPPM_DPLL_STAT, &polldata.value);
+        PK_TRACE_INF("Poll on DPLL_STAT[freq_change=0]");
+
+        // ... to indicate that the DPLL is safely either at the new frequency
+        // or in droop protection below the new frequency
+        do
+        {
+            ippm_read(QPPM_DPLL_STAT, &data.value);
+        }
+        while((data.words.lower & BIT64SH(61)));
+    }
+
+    // DPLL Mode 3
+    if(cme_flags & BIT32(CME_FLAGS_VDM_OPERABLE))
+    {
+        PK_TRACE_INF("Poll on DPLL_STAT[update_complete=1]");
+        // ... to indicate that the DPLL has sampled the newly requested
+        // frequency into its internal registers as a target,
+        // but may not yet be there
+        uint32_t tbStart, tbEnd, elapsed;
+        cppm_csar_t csar;
+        csar.value = 0;
+
+        //Read CPPM_CSAR
+        if (cme_flags & BIT32(CME_FLAGS_CORE0_GOOD))
+        {
+            CME_GETSCOM(CPPM_CSAR, CME_MASK_C0, csar.value);
+        }
+        else if (cme_flags & BIT32(CME_FLAGS_CORE1_GOOD))
+        {
+            CME_GETSCOM(CPPM_CSAR, CME_MASK_C1, csar.value);
+        }
+
+        //Read TimebaseStart
+        tbStart = in32(CME_LCL_TBR);
+
+        do
+        {
+            //Read DPLL_STAT
+            ippm_read(QPPM_DPLL_STAT, &data.value);
+
+            //If QPPM_DPLL_STAT[60/UPDATE_COMPLETE]=1
+            if(data.words.lower & BIT64SH(60))
+            {
+                break;
+            }
+
+            //Read TimebaseEnd
+            tbEnd = in32(CME_LCL_TBR);
+
+            //Compute Elapsed Count with accounting for Timebase Wrapping
+            if (tbEnd > tbStart)
+            {
+                elapsed = tbEnd - tbStart;
+            }
+            else
+            {
+                elapsed = 0xFFFFFFFF - tbStart + tbEnd + 1;
+            }
+
+            //If !CPPM_CSAR[DIS_NACK] AND (Elapsed Count > DROOP_POLL_COUNT), then prolonged
+            //droop detected
+            if (!(csar.value & BIT64(CPPM_CSAR_DISABLE_CME_NACK_ON_PROLONGED_DROOP)) &&
+                elapsed > DROOP_POLL_COUNT)
+            {
+                rc = 1; //Non-zero return code
+                break;
+            }
+        }
+        while(1);
     }
-    while(!(polldata.words.lower & BIT32(28)));
+
+    return rc;
 }
 
 // Non-atomic Interppm-read, this function is not made availabe via the header
@@ -437,8 +505,9 @@ uint32_t calc_vdm_jump_values(uint32_t pstate, uint32_t region)
     return new_jump_values;
 }
 
-void update_vdm_jump_values_in_dpll(uint32_t pstate, uint32_t region)
+uint32_t update_vdm_jump_values_in_dpll(uint32_t pstate, uint32_t region)
 {
+    uint32_t rc = 0;
     data64_t scom_data = { 0 };
     uint32_t new_jump_values = calc_vdm_jump_values(pstate, region);
     // Read the current contents of DPLL_CTRL and then update only the jump
@@ -472,38 +541,44 @@ void update_vdm_jump_values_in_dpll(uint32_t pstate, uint32_t region)
         reduced_dpll_val.fields.fmin = reduced_dpll_val.fields.fmult;
         // Write the reduced frequency
         nonatomic_ippm_write(QPPM_DPLL_FREQ, reduced_dpll_val.value);
-        poll_dpll_update_complete();
-        // Clear jump enable (drop to Mode 2)
-        nonatomic_ippm_write(QPPM_DPLL_CTRL_CLR, BIT64(1));
-        // Poll for lock
-        PK_TRACE_INF("Poll on DPLL_STAT[block_active|lock]");
+        rc = poll_dpll_stat();
 
-        // ... to indicate that the DPLL is safely either at the new frequency
-        // or in droop protection below the new frequency
-        do
+        if (!rc)
         {
-            nonatomic_ippm_read(QPPM_DPLL_STAT, &poll_data.value);
-        }
-        while(!(poll_data.words.lower & BITS32(30, 2)));
-
-        // Write the new jump values (clear jump enable)
-        scom_data.value &= ~BIT64(1);
-        scom_data.words.lower = new_jump_values;
-        nonatomic_ippm_write(QPPM_DPLL_CTRL, scom_data.value);
-        // Set jump enable (switch back to Mode 3)
-        nonatomic_ippm_write(QPPM_DPLL_CTRL_OR, BIT64(1));
-
-        // The frequency will be raised as part of the pstate transition if
-        // lowering the pstate, don't need to do anything here
-        if(pstate >= G_cme_pstate_record.quadPstate)
-        {
-            // Restore frequency
-            nonatomic_ippm_write(QPPM_DPLL_FREQ, saved_dpll_val.value);
-            poll_dpll_update_complete();
+            // Clear jump enable (drop to Mode 2)
+            nonatomic_ippm_write(QPPM_DPLL_CTRL_CLR, BIT64(1));
+            // Poll for lock
+            PK_TRACE_INF("Poll on DPLL_STAT[block_active|lock]");
+
+            // ... to indicate that the DPLL is safely either at the new frequency
+            // or in droop protection below the new frequency
+            do
+            {
+                nonatomic_ippm_read(QPPM_DPLL_STAT, &poll_data.value);
+            }
+            while(!(poll_data.words.lower & BITS32(30, 2)));
+
+            // Write the new jump values (clear jump enable)
+            scom_data.value &= ~BIT64(1);
+            scom_data.words.lower = new_jump_values;
+            nonatomic_ippm_write(QPPM_DPLL_CTRL, scom_data.value);
+            // Set jump enable (switch back to Mode 3)
+            nonatomic_ippm_write(QPPM_DPLL_CTRL_OR, BIT64(1));
+
+            // The frequency will be raised as part of the pstate transition if
+            // lowering the pstate, don't need to do anything here
+            if(pstate >= G_cme_pstate_record.quadPstate)
+            {
+                // Restore frequency
+                nonatomic_ippm_write(QPPM_DPLL_FREQ, saved_dpll_val.value);
+                rc = poll_dpll_stat();
+            }
         }
 
         pk_critical_section_exit(&ctx);
     }
+
+    return rc;
 }
 #endif//NIMBUS_DD_LEVEL
 
@@ -549,11 +624,11 @@ void calc_vdm_threshold_indices(uint32_t pstate, uint32_t region,
                               ? indices[VDM_SMALL_IDX] : indices[VDM_XTREME_IDX];
 }
 
-void p9_cme_vdm_update(uint32_t pstate)
+uint32_t p9_cme_vdm_update(uint32_t pstate)
 {
     // Static forces this array into .sbss instead of calling memset()
     static uint32_t new_idx[NUM_THRESHOLD_POINTS] = { 0 };
-    uint32_t i = 0;
+    uint32_t i = 0, rc = 0;
     // Set one bit per threshold starting at bit 31 (28,29,30,31)
     uint32_t not_done = BITS32(32 - NUM_THRESHOLD_POINTS, NUM_THRESHOLD_POINTS);
     uint64_t scom_data = 0;
@@ -603,8 +678,9 @@ void p9_cme_vdm_update(uint32_t pstate)
     while(not_done);
 
 #if NIMBUS_DD_LEVEL != 10
-    update_vdm_jump_values_in_dpll(pstate, region);
+    rc = update_vdm_jump_values_in_dpll(pstate, region);
 #endif//NIMBUS_DD_LEVEL
+    return rc;
 }
 #endif//USE_CME_VDM_FEATURE
 
@@ -712,6 +788,12 @@ void p9_cme_pstate_pmsr_updt(uint32_t coreMask)
                 pmsrData |= BIT64(33);
             }
 
+            //PMSR[35] is PSTATES_SUSPENDED bit
+            if(G_cme_pstate_record.pstatesSuspended)
+            {
+                pmsrData |= BIT64(35);
+            }
+
             out64(CME_LCL_PMSRS0 + ((cm & 0x1) << 5), pmsrData);
         }
     }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
index 8fabc5b0..b13ca15f 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
@@ -57,18 +57,24 @@ enum DPLL
     DPLL_MAX_VALUE = 300, //DPLL value @5000Mhz (5000/16.667)
 };
 
+enum DROOP_POLL
+{
+    DROOP_POLL_TIME_NS = 200000, //This defines the duration
+    DROOP_POLL_COUNT = DROOP_POLL_TIME_NS / 32 //32ns per timebase tick
+};
+
 typedef struct
 {
-    uint32_t core0_resclk_idx;
-    uint32_t core1_resclk_idx;
-    uint32_t l2_ex0_resclk_idx;
-    uint32_t l2_ex1_resclk_idx;
-    uint32_t common_resclk_idx;
+    uint32_t core0_resclk_idx;  //4
+    uint32_t core1_resclk_idx;  //8
+    uint32_t l2_ex0_resclk_idx; //12
+    uint32_t l2_ex1_resclk_idx; //16
+    uint32_t common_resclk_idx; //20
 } cme_resclk_data_t;
 
 typedef struct
 {
-    uint32_t vdm_threshold_idx[NUM_THRESHOLD_POINTS];
+    uint32_t vdm_threshold_idx[NUM_THRESHOLD_POINTS]; //4*4=16
 } cme_vdm_data_t;
 
 typedef enum
@@ -160,20 +166,21 @@ typedef struct
 #if !defined(__IOTA__)
     PkSemaphore sem[2];
 #endif
-    uint32_t qmFlag;
-    uint32_t siblingCMEFlag;
-    uint32_t quadPstate;
-    uint32_t cmeMaskGoodCore;
-    uint32_t globalPstate;
+    uint32_t qmFlag;            //4
+    uint32_t siblingCMEFlag;    //8
+    uint32_t quadPstate;        //12
+    uint32_t cmeMaskGoodCore;   //16
+    uint32_t globalPstate;      //20
 #ifdef USE_CME_RESCLK_FEATURE
-    cme_resclk_data_t resclkData;
+    cme_resclk_data_t resclkData; //40
 #endif//USE_CME_RESCLK_FEATURE
 #ifdef USE_CME_VDM_FEATURE
-    cme_vdm_data_t vdmData;
+    cme_vdm_data_t vdmData; //56
 #endif//USE_CME_VDM_FEATURE
-    uint32_t pmin;
-    uint32_t safeMode;
-    uint32_t pmax;
+    uint32_t pmin;          //60
+    uint32_t safeMode;      //64
+    uint32_t pmax;          //68
+    uint32_t pstatesSuspended; //72(0x48)
 } CmePstateRecord;
 
 typedef struct
@@ -182,14 +189,17 @@ typedef struct
 } cme_pstate_pmcr_data_t;
 
 void p9_cme_pstate_pmcr_thread(void*);
-void p9_cme_pstate_db_thread(void*);
+void p9_cme_pstate_db0_thread(void*);
 void p9_cme_pstate_pmcr_handler(void*, PkIrqId);
-void p9_cme_pstate_db_handler(void*, PkIrqId);
+void p9_cme_pstate_db0_handler(void*, PkIrqId);
+void p9_cme_pstate_db3_handler(void*, PkIrqId);
+void p9_cme_pstate_db3_handler_replay_db0();
+void p9_cme_pstate_db3_handler_high_priority_pstate(uint32_t pstate);
 void p9_cme_pstate_intercme_in0_irq_handler(void*, PkIrqId);
 void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq);
 void p9_cme_pstate_db0_safe_mode();
 int send_pig_packet(uint64_t data, uint32_t coreMask);
-void poll_dpll_update_complete();
+uint32_t poll_dpll_stat();
 void ippm_read(uint32_t addr, uint64_t* data);
 void ippm_write(uint32_t addr, uint64_t data);
 void intercme_msg_send(uint32_t msg, INTERCME_MSG_TYPE type);
@@ -198,7 +208,8 @@ void intercme_direct(INTERCME_DIRECT_INTF intf, INTERCME_DIRECT_TYPE type, uint3
 void p9_cme_core_stop_analog_control(uint32_t core_mask, ANALOG_CONTROL enable);
 void p9_cme_pstate_pmsr_updt(uint32_t coreMask);
 void p9_cme_pstate_pmsr_updt_in_progress(uint32_t coreMask);
-void p9_cme_pstate_intercme_in0_handler();
+void p9_cme_pstate_sibling_lock_and_intercme_protocol(uint32_t process_intercme_in0, uint32_t readDB0, uint64_t dbData);
+void p9_cme_pstate_process_db0_sibling(uint64_t dbData);
 
 #ifdef USE_CME_RESCLK_FEATURE
     uint32_t p9_cme_resclk_get_index(uint32_t pstate);
@@ -206,8 +217,8 @@ void p9_cme_pstate_intercme_in0_handler();
 #endif//USE_CME_RESCLK_FEATURE
 #ifdef USE_CME_VDM_FEATURE
 uint32_t calc_vdm_jump_values(uint32_t pstate, uint32_t region);
-void update_vdm_jump_values_in_dpll(uint32_t pstate, uint32_t region);
-void p9_cme_vdm_update(uint32_t pstate);
+uint32_t update_vdm_jump_values_in_dpll(uint32_t pstate, uint32_t region);
+uint32_t p9_cme_vdm_update(uint32_t pstate);
 uint32_t pstate_to_vid_compare(uint32_t pstate, uint32_t region);
 uint32_t pstate_to_vpd_region(uint32_t pstate);
 void calc_vdm_threshold_indices(uint32_t pstate, uint32_t region,
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
index 49e946e5..59c1b1ea 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_pstate.c
@@ -34,6 +34,7 @@
 #include "wof_sgpe_pgpe_api.h"
 #include "p9_pgpe_header.h"
 #include "p9_pgpe_optrace.h"
+#include "ppe42_cache.h"
 
 //
 //#Defines
@@ -51,6 +52,7 @@ extern uint32_t G_ext_vrm_inc_rate_mult_usperus;
 extern uint32_t G_ext_vrm_dec_rate_mult_usperus;
 extern PgpePstateRecord G_pgpe_pstate_record;
 extern void p9_pgpe_ipc_ack_sgpe_ctrl_stop_updt(ipc_msg_t* msg, void* arg);
+extern void p9_pgpe_ipc_ack_sgpe_suspend_stop(ipc_msg_t* msg, void* arg);
 
 //
 //Global Data
@@ -61,8 +63,9 @@ GPE_BUFFER(ipcmsg_p2s_suspend_stop_t G_sgpe_suspend_stop);
 
 //Local Functions
 void p9_pgpe_pstate_freq_updt();
-void p9_pgpe_suspend_stop_callback(ipc_msg_t* msg, void* arg);
 void p9_pgpe_pstate_dpll_write(uint32_t quadsVector, uint64_t val);
+void p9_pgpe_droop_throttle();
+void p9_pgpe_droop_unthrottle();
 
 //
 //p9_pgpe_pstate_init
@@ -438,12 +441,16 @@ void p9_pgpe_pstate_updt_actual_quad(uint32_t quadsVector)
 //p9_pgpe_send_db0
 //
 //Sends DB0 to CMEs
-void p9_pgpe_send_db0(uint64_t db0, uint32_t coreVector, uint32_t unicast, uint32_t ack, uint32_t ackVector)
+void p9_pgpe_send_db0(uint64_t db0, uint32_t origCoreVector,
+                      uint32_t unicast, uint32_t ackWait, uint32_t origAckVector)
 {
 
-    uint32_t c;
+    uint32_t c, q;
+    uint32_t ackVector = origAckVector;
+    uint32_t coreVector = origCoreVector;
+    G_pgpe_pstate_record.quadsNACKed = 0;
 
-    PK_TRACE_DBG("SDB: Send DB0");
+    PK_TRACE_DBG("SDB: Send DB0 coreVector=0x%x,unicast=0x%x,ackVector=0x%x", origCoreVector, unicast, origAckVector);
 
     //In case of unicast, only write DB0 for active cores. However, in case of
     //multicast just write DB0 of every configured core, but care only about active cores.
@@ -471,16 +478,94 @@ void p9_pgpe_send_db0(uint64_t db0, uint32_t coreVector, uint32_t unicast, uint3
 #endif
     }
 
-    if (ack == PGPE_DB0_ACK_WAIT_CME)
+    p9_pgpe_wait_cme_db_ack(ackVector);//Wait for ACKs from QuadManagers
+
+    PK_TRACE_DBG("SDB: quadsNACKed=0x%x", G_pgpe_pstate_record.quadsNACKed);
+
+    if(G_pgpe_pstate_record.quadsNACKed)
     {
-        p9_pgpe_wait_cme_db_ack(ackVector);//Wait for ACKs from QuadManagers
+        PK_TRACE_DBG("SDB: quadsNACKed=0x%x", G_pgpe_pstate_record.quadsNACKed);
+
+        //a. If OCC Scratch2 Core Throttle Continuous Change Enable bit is set (i.e. during Manufacturing test), halt the PGPE with a unique error code.
+        //Engineering Note: characterization team is responsible to set CSAR bit "Disable CME NACK on Prolonged Droop" when doing PGPE throttle scom injection.
+        if(in32(OCB_OCCS2) & BIT32(CORE_THROTTLE_CONTINUOUS_CHANGE_ENABLE))
+        {
+            PGPE_PANIC_AND_TRACE(PGPE_DROOP_AND_CORE_THRTLE_ENABLED);
+        }
+
+        //b) If  OCC flag PGPE Prolonged Droop Workaround Active bit is not set,
+        //    call droop_throttle()
+
+        if (!(in32(OCB_OCCFLG) & BIT32(PGPE_PROLONGED_DROOP_WORKAROUND_ACTIVE)))
+        {
+            p9_pgpe_droop_throttle();
+        }
+
+        //c) Send DB3 (Replay Previous DB0 Operation) to only the CME Quad Managers, and
+        //their Sibling CME (if present), that responded with a NACK.
+        uint64_t db3val = (uint64_t)MSGID_DB3_REPLAY_DB0 << 56;
+
+        while(G_pgpe_pstate_record.quadsNACKed)
+        {
+            G_pgpe_pstate_record.cntNACKs++;
+            ackVector = G_pgpe_pstate_record.quadsNACKed;
+
+            for (q = 0; q < MAX_QUADS; q++)
+            {
+                //If quad provided an ACK, then don't send DB3 again
+                if(!(G_pgpe_pstate_record.quadsNACKed & QUAD_MASK(q)))
+                {
+                    coreVector &= (~QUAD_ALL_CORES_MASK(q));
+                }
+                else
+                {
+                    G_pgpe_pstate_record.quadsCntNACKs[q]++;
+                }
+            }
+
+            //If a NACK received was in response to the first retry (i.e. second failed attempt):
+            if (G_pgpe_pstate_record.cntNACKs == 2)
+            {
+                // 1 SCOM Write to OCC FIR[prolonged_droop_detected] bit.   This FIR bit is set to recoverable so that it will create an informational error log.
+                GPE_PUTSCOM(OCB_OCCLFIR_OR, BIT64(OCCLFIR_PROLONGED_DROOP_DETECTED));
+
+                // 2 Set OCC Flag register PGPE PM Reset Suppress bit that OCC
+                //  will read to tell OCC not to attempt a PM Complex reset on
+                //  PGPE timeouts in the meantime.
+                out32(OCB_OCCFLG_OR, BIT32(PGPE_PM_RESET_SUPPRESS));
+
+                // 3 Send DB0 PMSR Update with message Set Pstates Suspended only
+                // to the CME QM (and their Siblings) that provided an ACK
+                // (note: PGPE must also wait for them to ACK the DB0)
+                p9_pgpe_pstate_pmsr_updt(DB0_PMSR_UPDT_SET_PSTATES_SUSPENDED,
+                                         origCoreVector & (~coreVector),
+                                         origAckVector & (~G_pgpe_pstate_record.quadsNACKed));
+            }
+
+            //The PGPE then retries the DB3 (Replay Previous DB0 Operation)
+            //again as described above to all CME QM (and their Siblings)
+            //that responded with NACK until it no longer gets a NACK (attempt to self-heal)
+            for (c = 0; c < MAX_CORES; c++)
+            {
+                if (coreVector & CORE_MASK(c))
+                {
+                    GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB3, c), db3val)
+                }
+            }
+
+            p9_pgpe_wait_cme_db_ack(ackVector);//Wait for ACKs from QuadManagers
+        }//End while(quadNACked) loop
+
+        //if OCC Flag Register PGPE Prolonged Droop Workaround Active bit is set and all CME QMs respond with ACK
+        p9_pgpe_droop_unthrottle();
     }
 }
 
 void p9_pgpe_wait_cme_db_ack(uint32_t quadAckExpect)
 {
-    uint32_t q;
+    uint32_t q, c;
     uint32_t opit4pr, opit4prQuad, opit4Clr = 0;
+    ocb_opit4cn_t opit4cn;
 
     PK_TRACE_INF("DBW: AckExpect=0x%x", quadAckExpect);
 
@@ -498,9 +583,45 @@ void p9_pgpe_wait_cme_db_ack(uint32_t quadAckExpect)
             {
                 if (quadAckExpect & QUAD_MASK(q))
                 {
+                    PK_TRACE_INF("DBW: Quad[%d] Acked", q);
                     quadAckExpect &= ~QUAD_MASK(q);
                     opit4Clr |= (opit4prQuad << ((MAX_QUADS - q + 1) << 2));
-                    PK_TRACE_DBG("DBW: GotAck from %d", q);
+                    c = 0;
+
+                    while(!(opit4prQuad & (0x8 >> c)))
+                    {
+                        c++;
+                    }
+
+                    opit4cn.value = in32(OCB_OPIT4CN((q * 4) + c));
+
+                    PK_TRACE_INF("DBW: opit4cn[%d]=0x%x", c, opit4cn.value);
+
+                    switch (opit4cn.value & 0xf)
+                    {
+                        case MSGID_PCB_TYPE4_ACK_ERROR: //0x0
+                            PGPE_PANIC_AND_TRACE(PGPE_CME_DB0_ERROR_ACK);
+                            break;
+
+                        case MSGID_PCB_TYPE4_ACK_PSTATE_PROTO_ACK: //0x1
+                        case MSGID_PCB_TYPE4_ACK_PSTATE_SUSPENDED: //0x2
+                            //Do nothing as there is no error
+                            break;
+
+                        case MSGID_PCB_TYPE4_NACK_DROOP_PRESENT: //0x4
+                            //Mark that this quad sent a NACK
+                            PK_TRACE_DBG("DBW: Got NACK from %d", q);
+                            G_pgpe_pstate_record.quadsNACKed |= QUAD_MASK(q);
+                            PK_TRACE_DBG("DBW: QuadsNACKed=0x%x", G_pgpe_pstate_record.quadsNACKed);
+                            break;
+
+                        //Note this includes MSGID_PCB_TYPE4_QUAD_MGR_AVAILABLE(0x3)
+                        //and other undefined encoding
+                        default:
+                            PK_TRACE_ERR("DBW:Unexpected qCME[%u] ACK type", q);
+                            PGPE_PANIC_AND_TRACE(PGPE_CME_UNEXPECTED_DB0_ACK);
+                    }
+
                 }
                 else if(!(G_pgpe_pstate_record.pendQuadsRegisterReceive & QUAD_MASK(q)))
                 {
@@ -513,7 +634,7 @@ void p9_pgpe_wait_cme_db_ack(uint32_t quadAckExpect)
         out32(OCB_OPIT4PRA_CLR, opit4Clr);
     }
 
-    PK_TRACE_DBG("DBW:qCME ACKs rcvd");
+    PK_TRACE_DBG("DBW: QuadsNACKed=0x%x", G_pgpe_pstate_record.quadsNACKed);
 }
 
 //
@@ -730,6 +851,11 @@ void p9_pgpe_pstate_start(uint32_t pstate_start_origin)
                      PGPE_DB0_ACK_WAIT_CME,
                      G_pgpe_pstate_record.activeQuads);
 
+    //In case VDM Prolonged Droop event occured during PSTATE_START, then clearing
+    //ensures OCC is notified about Prolonged Droop event resolution.
+    //Also, at this point nothing else should be pending from OCC, so safe to clear.
+    out32(OCB_OCCFLG_CLR, BIT32(PGPE_PM_RESET_SUPPRESS));
+
     //Lower voltage if boot voltage > syncPstate voltage
     if (G_pgpe_pstate_record.eVidCurr > G_pgpe_pstate_record.eVidNext)
     {
@@ -865,9 +991,8 @@ void p9_pgpe_pstate_stop()
                 GPE_PUTSCOM(GPE_SCOM_ADDR_CME(CME_SCOM_LMCR_OR, q, 1), BIT64(0));
             }
 
-            //PGPE opens DPLL for SCOMs on quads that are about to enter STOP11
-            //or have exited STOP11, but not registered yet. Or, to said another
-            //non-active quads that are powered on will have DPLL open for SCOMs
+            //Only need to do that for active quads here because PGPE opens DPLL for SCOMs
+            //for other quads.
             GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, q), BIT64(26));
         }
     }
@@ -1218,14 +1343,9 @@ void p9_pgpe_pstate_safe_mode()
     PK_TRACE_INF("SAF: Safe Mode Actuation Done!");
 
     //Send SAFE Mode Bcast to all CMEs
-    pgpe_db0_safe_mode_bcast_t db0_sm_bcast;
-    db0_sm_bcast.value = 0;
-    db0_sm_bcast.fields.msg_id = MSGID_DB0_SAFE_MODE_BROADCAST;
-    p9_pgpe_send_db0(db0_sm_bcast.value,
-                     G_pgpe_pstate_record.activeDB,
-                     PGPE_DB0_UNICAST,
-                     PGPE_DB0_ACK_WAIT_CME,
-                     G_pgpe_pstate_record.activeQuads);
+    p9_pgpe_pstate_pmsr_updt(DB0_PMSR_UPDT_SET_SAFE_MODE,
+                             G_pgpe_pstate_record.activeDB,
+                             G_pgpe_pstate_record.activeQuads);
 
     //Update PstatesStatus to PM_SUSPEND_PENDING or PSTATE_SAFE_MODE
     G_pgpe_pstate_record.pstatesStatus = suspend ? PSTATE_PM_SUSPEND_PENDING : PSTATE_SAFE_MODE;
@@ -1297,31 +1417,75 @@ void p9_pgpe_pstate_safe_mode()
 //
 //p9_pgpe_pstate_send_suspend_stop
 //
-void p9_pgpe_pstate_send_suspend_stop()
+void p9_pgpe_pstate_send_suspend_stop(uint32_t command)
 {
-#if SGPE_IPC_ENABLED == 1
+//#if SGPE_IPC_ENABLED == 1
     p9_pgpe_optrace(PRC_PM_SUSP);
 
     int rc;
     G_sgpe_suspend_stop.fields.msg_num = MSGID_PGPE_SGPE_SUSPEND_STOP;
+    G_sgpe_suspend_stop.fields.command  = command;
     G_sgpe_suspend_stop.fields.return_code = 0x0;
     G_ipc_msg_pgpe_sgpe.cmd_data = &G_sgpe_suspend_stop;
     ipc_init_msg(&G_ipc_msg_pgpe_sgpe.cmd,
                  IPC_MSGID_PGPE_SGPE_SUSPEND_STOP,
-                 p9_pgpe_suspend_stop_callback,
+                 p9_pgpe_ipc_ack_sgpe_suspend_stop,
                  (void*)NULL);
 
-    //Set SCOM Ownership of PMCR
-    PK_TRACE_INF("SUSP: Setting SCOM Ownership of PMCRs");
+    //send the command
+    PK_TRACE_INF("SUSP:Sending Suspend Stop(cmd=0x%x)", command);
+    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
 
+    if(rc)
+    {
+        PK_TRACE_ERR("SUSP:Suspend Stop BAD ACK");
+        PGPE_PANIC_AND_TRACE(PGPE_SGPE_SUSPEND_STOP_BAD_ACK);
+    }
+
+    //Just block until SGPE writes the return code field
+    //It is assumed that SGPE will write this field right before
+    //calling ipc_send_rsp(). Also, note that PGPE will get the ACK
+    //in the form of IPC interrupt which will call the ipc_ack_suspend_stop
+    //This IPC handler is effectively a NOP writing the return code fields
+    //act as ACK for PGPE. Alternative is to wait until ACKs arrives and do processing
+    //in the IPC ACK handler, but this requires opening up IPC interrupt. Instead, we
+    //do processing after blocking
+    while(G_sgpe_suspend_stop.fields.return_code == IPC_SGPE_PGPE_RC_NULL)
+    {
+        dcbi(((void*)(&G_sgpe_suspend_stop)));
+    }
+
+    if (G_sgpe_suspend_stop.fields.return_code != IPC_SGPE_PGPE_RC_SUCCESS)
+    {
+        PK_TRACE_ERR("ERROR: SGPE Suspend STOP Bad RC. Halting PGPE!");
+        PGPE_PANIC_AND_TRACE(PGPE_SGPE_SUSPEND_STOP_BAD_ACK);
+    }
+
+    PK_TRACE_INF("SUSP:Suspend Stop(cmd=0x%x) ACKed", command);
+}
+
+//
+//This function sends Suspend Stop(Entry and Exit) to SGPE
+//It is called as part of PM Complex Suspend processing. Before
+//calling this function it is assumed the system is in SAFE_MODE.
+//There is no function to undo the suspend. Only PM Reset can bring the
+//PM Complex out of this state.
+//
+void p9_pgpe_pstate_pm_complex_suspend()
+{
     int q = 0;
     ocb_qcsr_t qcsr;
+
+    //Send IPC and wait for ACK
+    p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_SUSPEND_ENTRY_EXIT);
+
+    //Change PMCR ownership
+    PK_TRACE_INF("SUSP: Setting SCOM Ownership of PMCRs");
     qcsr.value = in32(OCB_QCSR);
 
     //Set LMCR for each CME
     for (q = 0; q < MAX_QUADS; q++)
     {
-        //if(G_pgpe_pstate_record.pReqActQuads->fields.requested_active_quads & (0x80 >> q))
         if(G_pgpe_pstate_record.activeQuads & QUAD_MASK(q))
         {
             //CME0 within this quad
@@ -1338,27 +1502,7 @@ void p9_pgpe_pstate_send_suspend_stop()
         }
     }
 
-    //send the command
-    PK_TRACE_INF("SUSP:Sending Suspend  IPC to SGPE");
-    rc = ipc_send_cmd(&G_ipc_msg_pgpe_sgpe.cmd);
-
-    if(rc)
-    {
-        PK_TRACE_ERR("SUSP:Suspend Stop BAD ACK");
-        PGPE_PANIC_AND_TRACE(PGPE_SGPE_SUSPEND_STOP_BAD_ACK);
-    }
-
-#else
-    p9_pgpe_suspend_stop_callback(NULL, NULL);
-#endif
-}
-
-//
-//p9_pgpe_suspend_stop_callback
-//
-void p9_pgpe_suspend_stop_callback(ipc_msg_t* msg, void* arg)
-{
-    PK_TRACE_INF("SUSP:Stop Cb");
+    //OP Trace and Set OCCS2[PM_COMPLEX_SUSPENDED)
     p9_pgpe_optrace(ACK_PM_SUSP);
     uint32_t occScr2 = in32(OCB_OCCS2);
     occScr2 |= BIT32(PM_COMPLEX_SUSPENDED);
@@ -1366,6 +1510,21 @@ void p9_pgpe_suspend_stop_callback(ipc_msg_t* msg, void* arg)
     out32(OCB_OCCS2, occScr2);
 }
 
+
+void p9_pgpe_pstate_pmsr_updt(uint32_t command, uint32_t targetCoresVector, uint32_t quadsAckVector)
+{
+    pgpe_db0_pmsr_updt_t db0_pmsr_updt;
+    db0_pmsr_updt.value = 0;
+    db0_pmsr_updt.fields.msg_id = MSGID_DB0_PMSR_UPDT;
+    db0_pmsr_updt.fields.command = command;
+    p9_pgpe_send_db0(db0_pmsr_updt.value,
+                     targetCoresVector,
+                     PGPE_DB0_UNICAST,
+                     PGPE_DB0_ACK_WAIT_CME,
+                     quadsAckVector);
+}
+
+
 //
 //p9_pgpe_pstate_at_target
 //
@@ -1702,3 +1861,96 @@ void p9_pgpe_pstate_ipc_rsp_cb_sem_post(ipc_msg_t* msg, void* arg)
 {
     pk_semaphore_post((PkSemaphore*)arg);
 }
+
+// Write to the PC Throttle Control register.  Ignore any PCB errors.
+// Option to retry on known PC hardware bug (always cleanup if happens)
+//
+void p9_pgpe_pstate_write_core_throttle(uint32_t throttleData, uint32_t enable_retry)
+{
+    uint32_t pc_fail;
+    uint32_t value = mfmsr();
+    mtmsr(value | MSR_THROTTLE_MASK); //don't cause halt if all cores offline or address error (PC Timeout)
+
+    do
+    {
+        out64(THROTTLE_SCOM_MULTICAST_WRITE, ((uint64_t) throttleData << 32)); //apply new throttle SCOM setting
+
+        // Work-around for PC HW problem. See SW407201
+        // If ADDRESS_ERROR then perform a SCOM write of all zeros to
+        // 2n010800 where n is the core number. Ignore ADDRESS_ERROR
+        // and TIMEOUT returned.
+        pc_fail = (((mfmsr() >> 20) & 0x7) == 0x4);
+
+        if (pc_fail)
+        {
+            out64(WORKAROUND_SCOM_MULTICAST_WRITE, 0);
+        }
+    }
+    while (enable_retry && pc_fail);
+
+    mtmsr(value); // restore MSR
+}
+
+void p9_pgpe_droop_throttle()
+{
+    PK_TRACE_DBG("DTH: Droop Throttle Enter");
+
+    uint32_t q;
+    //1.  PGPE sends IPC to SGPE to Suspend Stop Entries Only, and poll for Success return code (do not open IPCs, ignore the subsequent ACK).  This will prevent a core from going into Stop2 and missing the subsequent "unthrottle."
+    //p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_SUSPEND_ENTRY);
+
+
+    //2.  Call the core_instruction_throttle() procedure to enable throttle (same as used by FIT).
+    p9_pgpe_pstate_write_core_throttle(CORE_IFU_THROTTLE, RETRY);
+
+    //3.  Set the OCC flag PGPE Prolonged Droop Workaround Active bit.
+    out32(OCB_OCCFLG_OR, BIT32(PGPE_PROLONGED_DROOP_WORKAROUND_ACTIVE));
+
+    //4.  Clear the Prolonged Droop Global variables (Bit vector and retry counts).
+    G_pgpe_pstate_record.cntNACKs = 0;
+
+    for (q = 0; q < MAX_QUADS; q++)
+    {
+        G_pgpe_pstate_record.quadsCntNACKs[q] = 0;
+    }
+
+    //5.  Write PK Trace and Optrace record that the Prolonged Throttle workaround was engaged, including a bit vector of Quads that provided the NACK(s).
+    G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.activeQuads << 24)  |
+                                  (G_pgpe_pstate_record.globalPSCurr << 16) |
+                                  (G_pgpe_pstate_record.globalPSTarget << 8) |
+                                  G_pgpe_pstate_record.quadsNACKed;
+    p9_pgpe_optrace(PROLONGED_DROOP_EVENT);
+
+
+    PK_TRACE_DBG("DTH: Droop Throttle Exit");
+}
+
+
+void p9_pgpe_droop_unthrottle()
+{
+    PK_TRACE_DBG("DTH: Droop Unthrottle Enter");
+
+    //1.  Call the core_instruction_throttle() procedure to disable throttle (same as used by FIT).
+    p9_pgpe_pstate_write_core_throttle(CORE_THROTTLE_OFF, RETRY);
+
+    //2.  PGPE sends IPC to SGPE to Unsuspend STOP entries & poll for Success return code (do not open IPCs, ignore the subsequent ACK).
+    // p9_pgpe_pstate_send_suspend_stop(SUSPEND_STOP_UNSUSPEND_ENTRY);
+
+    //3.  Send Doorbell0 PMSR Update with message Clear Pstates Suspended to all configured cores in the active Quads.
+    p9_pgpe_pstate_pmsr_updt(DB0_PMSR_UPDT_CLEAR_PSTATES_SUSPENDED,
+                             G_pgpe_pstate_record.activeDB,
+                             G_pgpe_pstate_record.activeQuads);
+
+    //4.  Clear the OCC flag PGPE Prolonged Droop Workaround Active. OCCFLG[PM_RESET_SUPPRESS] will be cleared later
+    //after any pending IPCs from OCC have been processed and acked.
+    out32(OCB_OCCFLG_CLR, BIT32(PGPE_PROLONGED_DROOP_WORKAROUND_ACTIVE));
+
+    //5.  Write PK Trace and Optrace record that the Prolonged Throttle workaround was removed,
+    //including the Total Retry Count and the most recent bit vector of Quads that provided the NACK(s) .
+    G_pgpe_optrace_data.word[0] = (G_pgpe_pstate_record.quadsNACKed << 24)  |
+                                  (G_pgpe_pstate_record.activeCores);
+    G_pgpe_optrace_data.word[0] = G_pgpe_pstate_record.cntNACKs;
+    p9_pgpe_optrace(PROLONGED_DROOP_RESOLVED);
+
+    PK_TRACE_DBG("DTH: Droop Unthrottle Exit");
+}
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
index 541ee0ba..4ae8e342 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_actuate_pstates.c
@@ -105,6 +105,26 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
         //Loop while Pstate is ACTIVE
         while(G_pgpe_pstate_record.pstatesStatus == PSTATE_ACTIVE)
         {
+            //If a VDM prolonged droop happened, then we set OCB_OCCFLG[PGPE_PM_RESET_SUPPRESS]
+            //It should be cleared once VDM prolonged droop condition has subsided and all pending IPCs
+            //from OCC have been processed and acked. Note, that pending processing and pending ack are
+            //only set inside IPC handler, and it's possible that while PGPE is stuck in the VDM prolonged
+            //droop loop(the p9_pgpe_pstate_do_step function call) an IPC interrupt happened, so
+            //PGPE must be given a change to take IPC interrupt and see if any other IPC from OCC
+            //needs processing.
+            if ((in32(OCB_OCCFLG) & BIT32(PGPE_PM_RESET_SUPPRESS)))
+            {
+                if ((G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_ack == 0) &&
+                    (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_ack == 0)  &&
+                    (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_ack == 0)  &&
+                    (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_CLIP_UPDT].pending_processing == 0) &&
+                    (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_VFRT].pending_processing == 0)  &&
+                    (G_pgpe_pstate_record.ipcPendTbl[IPC_PEND_WOF_CTRL].pending_processing == 0))
+                {
+                    out32(OCB_OCCFLG_CLR, BIT32(PGPE_PM_RESET_SUPPRESS));
+                }
+            }
+
             //Actuate step(if needed)
             if(p9_pgpe_pstate_at_target() == 0)
             {
@@ -286,7 +306,7 @@ void p9_pgpe_thread_actuate_pstates(void* arg)
         //This check must come after the possible p9_pgpe_pstate_safe_mode() call above
         if (G_pgpe_pstate_record.pstatesStatus == PSTATE_PM_SUSPEND_PENDING)
         {
-            p9_pgpe_pstate_send_suspend_stop();
+            p9_pgpe_pstate_pm_complex_suspend();
         }
 
         if (G_pgpe_pstate_record.pstatesStatus == PSTATE_STOP_PENDING)
diff --git a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
index 33cd1b2f..48a44233 100644
--- a/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
+++ b/import/chips/p9/procedures/ppe_closed/pgpe/pstate_gpe/p9_pgpe_thread_process_requests.c
@@ -53,6 +53,7 @@ void p9_pgpe_process_wof_vfrt();
 void p9_pgpe_process_set_pmcr_req();
 void p9_pgpe_process_registration();
 void p9_pgpe_process_ack_sgpe_ctrl_stop_updt();
+void p9_pgpe_process_ack_sgpe_suspend_stop();
 
 //
 //Process Request Thread
@@ -167,11 +168,11 @@ void p9_pgpe_thread_process_requests(void* arg)
         if (G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_processing)
         {
             p9_pgpe_process_ack_sgpe_ctrl_stop_updt();
+        }
 
-            if(G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_ack == 1)
-            {
-                restore_irq = 0;
-            }
+        if (G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_SUSPEND_STOP].pending_processing)
+        {
+            p9_pgpe_process_ack_sgpe_suspend_stop();
         }
 
         if (G_pgpe_pstate_record.pendQuadsRegisterProcess != 0)
@@ -905,8 +906,7 @@ void p9_pgpe_process_registration()
     ccsr.value = in32(OCB_CCSR);
     ocb_qcsr_t qcsr;
     qcsr.value = in32(OCB_QCSR);
-    volatile uint32_t opit4pr;
-    uint32_t opit4prQuad, q, c, oldActiveDB, oldActiveQuads, unicastCoresVector = 0, quadsRegisterProcess;
+    uint32_t  q, c, oldActiveDB, oldActiveQuads, unicastCoresVector = 0, quadsRegisterProcess;
     uint32_t quadAckExpect = 0;
     uint64_t value;
     pgpe_db0_start_ps_bcast_t db0_glb_bcast;
@@ -1077,35 +1077,16 @@ void p9_pgpe_process_registration()
         db0_glb_bcast.fields.quad3_ps = G_pgpe_pstate_record.quadPSTarget[3];
         db0_glb_bcast.fields.quad4_ps = G_pgpe_pstate_record.quadPSTarget[4];
         db0_glb_bcast.fields.quad5_ps = G_pgpe_pstate_record.quadPSTarget[5];
-        p9_pgpe_send_db0(db0_glb_bcast.value, unicastCoresVector, PGPE_DB0_UNICAST, PGPE_DB0_ACK_SKIP, 0);
+        p9_pgpe_send_db0(db0_glb_bcast.value, unicastCoresVector, PGPE_DB0_UNICAST, PGPE_DB0_ACK_WAIT_CME, quadAckExpect);
 
-        //Now collect ACKS for Pstate Start DB0
-        while(quadAckExpect != 0)
+        //Quads
+        for (q = 0; q < MAX_QUADS; q++)
         {
-            opit4pr = in32(OCB_OPIT4PRA);
-
-            for (q = 0; q < MAX_QUADS; q++)
+            if (quadAckExpect & QUAD_MASK(q))
             {
-                opit4prQuad = (opit4pr >> ((MAX_QUADS - q + 1) << 2)) & 0xf;
-
-                if (opit4prQuad)
-                {
-                    //We expect type4 from this quad for the Pstate Start sent above
-                    if (quadAckExpect & QUAD_MASK(q))
-                    {
-                        quadAckExpect &= ~QUAD_MASK(q);
-                        out32(OCB_OPIT4PRA_CLR, opit4prQuad << ((MAX_QUADS - q + 1) << 2));
-                        PK_TRACE_DBG("PTH: Got DB0 Start ACK from %d", q);
-                        G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSTarget[q];
-                        G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
-                        p9_pgpe_pstate_updt_actual_quad(QUAD_MASK(q));
-                    }
-                    else if(!(G_pgpe_pstate_record.pendQuadsRegisterReceive & QUAD_MASK(q)))
-                    {
-                        PK_TRACE_ERR("PTH: Unexpected ACK q=0x%x,opit4prQuad=0x%x", q, opit4prQuad);
-                        PK_PANIC(PGPE_CME_UNEXPECTED_REGISTRATION);
-                    }
-                }
+                G_pgpe_pstate_record.quadPSCurr[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                G_pgpe_pstate_record.quadPSNext[q] = G_pgpe_pstate_record.quadPSTarget[q];
+                p9_pgpe_pstate_updt_actual_quad(QUAD_MASK(q));
             }
         }
     }
@@ -1118,3 +1099,9 @@ void p9_pgpe_process_ack_sgpe_ctrl_stop_updt()
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_processing = 0;
     G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_CTRL_STOP_UPDT].pending_ack = 0;
 }
+
+void p9_pgpe_process_ack_sgpe_suspend_stop()
+{
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_SUSPEND_STOP].pending_processing = 0;
+    G_pgpe_pstate_record.ipcPendTbl[IPC_ACK_SUSPEND_STOP].pending_ack = 0;
+}