first draft of ppe_closed/cme & sgpe, and common/pmlib/

Change-Id: Id2c21a8475ec9a10f73a864013fbdd3ea32a7650
Original-Change-Id: I08440fadfeff7e4777a776167d4fdcffe5fec82f
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/21058
Tested-by: Jenkins Server
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

3 files changed, 1367 insertions, 0 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
new file mode 100755
index 00000000..24c3ed87
--- /dev/null
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c
@@ -0,0 +1,679 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_entry.c $ */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2015,2018                                                    */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+/// \file p9_cme_stop_enter_thread.c
+/// \brief CME Stop Entey Thread
+///
+/// Features of this code file:
+/// - main() registers an IRQ handler which gets interrupted on a STOP update
+///   by a core.
+/// - main creates a thread which pends on semaphore to be posted by the IRQ
+///   handler.
+/// - p9_cme_stop_enter_thread() pends on semaphore from IRQ handler.
+/// - On interrupt, stop_handler() does as follows:
+///   - masks the IRQ
+///   - posts a semaphore
+///   - exits.
+/// - On semaphore post, p9_cme_stop_enter_thread() does as follows:
+///   - clears the IRQ
+///   - reads STOP
+///   - call p9_cme_stop_entry() sequence
+///   - unmasks the IRQ
+///   - pends on next semaphore post.
+///
+
+#include "p9_cme_stop.h"
+#include "p9_cme_stop_enter_marks.h"
+
+CmeStopRecord G_cme_stop_record = {0, 0, 0};
+
+int
+p9_cme_stop_entry()
+{
+
+    int             rc                  = 0;
+    uint8_t         entry_ongoing       = 1;
+    uint8_t         target_level;
+    uint8_t         deeper_level        = 0;
+    uint32_t        deeper_core         = 0;
+    uint32_t        core;
+    uint32_t        loop;
+    uint32_t        pm_states;
+    uint64_t        scom_data;
+    ppm_sshsrc_t    hist;
+    ppm_pig_t       pig;
+
+    //--------------------------------------------------------------------------
+    // BEGIN OF STOP ENTRY
+    //--------------------------------------------------------------------------
+
+    // First we need to determine which of the two STOP interrupts fired.
+    // Iow, which of the two cores, "left-0" or "right-1", updated their
+    // STOP PM_STATE. If both have fired by the time we get to this point,
+    // CME will do Daul-cast to both cores at the same time in entry flow.
+    core = (in32(CME_LCL_EISR) & BITS32(20, 2)) >> SHIFT32(21);
+    // override with partial good core mask
+    core = core & G_cme_stop_record.cme_flags & CME_MASK_BC;
+
+    PK_TRACE("SE0: Core Select[%d]", core);
+
+    // Return error if None of both fired
+    if (!core)
+    {
+        return CME_STOP_ENTRY_PM_NOT_ACTIVE;
+    }
+
+    //===================================
+    MARK_TAG(BEGINSCOPE_STOP_ENTRY, core)
+    //===================================
+
+    //--------------------------------------------------------------------------
+    // STOP LEVEL 1 (should be done by hardware)
+    //--------------------------------------------------------------------------
+
+    // Read SISR for pm_state_cX
+    pm_states = in32_sh(CME_LCL_SISR);
+    G_cme_stop_record.pm_state_c0 = (pm_states & BITS32(4, 4)) >> SHIFT32(7);
+    G_cme_stop_record.pm_state_c1 = (pm_states & BITS32(8, 4)) >> SHIFT32(11);
+#if LAB_MODE == 1
+
+    if (G_cme_stop_record.pm_state_c0 == STOP_LEVEL_11)
+    {
+        G_cme_stop_record.pm_state_c0 = STOP_LEVEL_9;
+    }
+
+    if (G_cme_stop_record.pm_state_c1 == STOP_LEVEL_11)
+    {
+        G_cme_stop_record.pm_state_c1 = STOP_LEVEL_9;
+    }
+
+#endif
+
+    // pm_active AND waken_up   : pm_state = sisr (new state)
+    // pm_active AND !waken_up  : pm_state = sisr (only with auto promote)
+    // !pm_active AND waken_up  : pm_state = 0    (state out of date)
+    // !pm_active AND !waken_up : pm_state = sisr (current state)
+    if (~core & G_cme_stop_record.cme_wakenup & CME_MASK_C0)
+    {
+        G_cme_stop_record.pm_state_c0 = 0;
+    }
+
+    if (~core & G_cme_stop_record.cme_wakenup & CME_MASK_C1)
+    {
+        G_cme_stop_record.pm_state_c1 = 0;
+    }
+
+    G_cme_stop_record.cme_wakenup &= ~core;
+
+    PK_TRACE("SE1: Stop Levels[%d %d]",
+             G_cme_stop_record.pm_state_c0, G_cme_stop_record.pm_state_c1);
+
+    // Return error if target STOP level == 1(Nap)
+    if((core == CME_MASK_C0 && G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_1) ||
+       (core == CME_MASK_C1 && G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_1) ||
+       (core == CME_MASK_BC && (G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_1 ||
+                                G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_1)))
+    {
+        return CME_STOP_ENTRY_STOP1_SENT_IRQ;
+    }
+
+    //--------------------------------------------------------------------------
+    // STOP LEVEL 2
+    //--------------------------------------------------------------------------
+
+    do   // while(0) loop for stop flow control
+    {
+
+        PK_TRACE("SE2.a");
+        // Disable fired Stop and corresponding Wakeup interrupts
+        out32(CME_LCL_EIMR_OR, (CME_MASK_BC << SHIFT32(21)) |
+              (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+
+        PK_TRACE("SE2.b");
+        // Also clear the status of the currently fired STOP interrupt(s)
+        out32(CME_LCL_EISR_CLR, core << SHIFT32(21));
+
+        PK_TRACE("SE2.c");
+        // Request PCB Mux
+        out32(CME_LCL_SICR_OR, core << SHIFT32(11));
+
+        PK_TRACE("SE2.d");
+        // Protect PPM Register Write
+        CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(0));
+
+        // set target_level from pm_state for both cores or just one core
+        target_level = (core == CME_MASK_C0) ? G_cme_stop_record.pm_state_c0 :
+                       G_cme_stop_record.pm_state_c1;
+
+        // If both cores are going into STOP but targeting different levels,
+        if ((core == CME_MASK_BC) &&
+            (G_cme_stop_record.pm_state_c0 != G_cme_stop_record.pm_state_c1))
+        {
+            // set target_level to the lighter level targeted by one core
+            // set deeper_level to the deeper level targeted by deeper core
+            deeper_level = G_cme_stop_record.pm_state_c0;
+            deeper_core  = CME_MASK_C0;
+
+            if (G_cme_stop_record.pm_state_c0 < G_cme_stop_record.pm_state_c1)
+            {
+                target_level = G_cme_stop_record.pm_state_c0;
+                deeper_level = G_cme_stop_record.pm_state_c1;
+                deeper_core  = CME_MASK_C1;
+            }
+        }
+
+        PK_TRACE("SE2.e");
+        // Update STOP History: In Transition of Entry
+        // Set req_stop_level to target_level of either both or just one core
+        CME_STOP_UPDATE_HISTORY(core,
+                                STOP_CORE_READY_RUN,
+                                STOP_TRANS_ENTRY,
+                                target_level,
+                                STOP_LEVEL_0,
+                                STOP_REQ_ENABLE,
+                                STOP_ACT_DISABLE);
+
+        // Set req_stop_level to deeper_level for deeper core
+        if (deeper_core)
+        {
+            CME_STOP_UPDATE_HISTORY(deeper_core,
+                                    STOP_CORE_READY_RUN,
+                                    STOP_TRANS_ENTRY,
+                                    deeper_level,
+                                    STOP_LEVEL_0,
+                                    STOP_REQ_ENABLE,
+                                    STOP_ACT_DISABLE);
+        }
+
+        PK_TRACE("SE2: target_lv[%d], deeper_lv[%d], deeper_core[%d]",
+                 target_level, deeper_level, deeper_core);
+
+        //=============================
+        MARK_TRAP(SE_POLL_PCBMUX_GRANT)
+        //=============================
+
+        PK_TRACE("SE2.f");
+
+        // Poll Infinitely for PCB Mux Grant
+        // MF: change watchdog timer in pk to ensure forward progress
+        while((core & (in32(CME_LCL_SISR) >> SHIFT32(11))) != core);
+
+        PK_TRACE("SE2: PCB Mux Granted");
+
+        //==========================
+        MARK_TRAP(SE_STOP_CORE_CLKS)
+        //==========================
+
+        PK_TRACE("SE2.g");
+        // Acknowledge the STOP Entry to PC with a pulse
+        out32(CME_LCL_SICR_OR, core << SHIFT32(1));
+        out32(CME_LCL_SICR_CLR, core << SHIFT32(1));
+
+        PK_TRACE("SE2.h");
+        // Raise Core-L2 + Core-CC Quiesces
+        out32(CME_LCL_SICR_OR, (core << SHIFT32(7)) | (core << SHIFT32(9)));
+
+        PK_TRACE("SE2.i");
+        // Waits quiesce done for at least 512 core cycles
+        // MF: verify generate FCB otherwise math is wrong.
+        CME_WAIT_CORE_CYCLES(loop, 512)
+
+        PK_TRACE("SE2.j");
+        // Raise Core Chiplet Fence
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(18));
+
+        PK_TRACE("SE2.k");
+        // Set all bits to zero prior stop core clocks
+        CME_PUTSCOM(C_SCAN_REGION_TYPE, core, 0);
+
+        PK_TRACE("SE2.l");
+        // Stop Core Clocks
+        CME_PUTSCOM(C_CLK_REGION, core, 0x9FFC00000000E000);
+
+        PK_TRACE("SE2.m");
+
+        // Poll for core clocks stopped
+        do
+        {
+            CME_GETSCOM(C_CLOCK_STAT_SL, core, CME_SCOM_AND, scom_data);
+        }
+        while((scom_data & BITS64(4, 10)) != BITS64(4, 10));
+
+        // MF: verify compiler generate single rlwmni
+        // MF: delay may be needed for stage latch to propagate thold
+
+        PK_TRACE("SE2: Core Clock Stopped");
+
+        //=========================
+        MARK_TRAP(SE_DROP_CLK_SYNC)
+        //=========================
+
+        PK_TRACE("SE2.n");
+        // Drop clock sync enable before switch to refclk
+        CME_PUTSCOM(CPPM_CACCR_CLR, core, BIT64(15));
+
+#if !EPM_P9_TUNING
+        PK_TRACE("SE2.o");
+
+        // Poll for clock sync done to drop
+        do
+        {
+            CME_GETSCOM(CPPM_CACSR, core, CME_SCOM_OR, scom_data);
+        }
+        while(scom_data & BIT64(13));
+
+#endif
+
+        PK_TRACE("SE2.p");
+        // Switch glsmux to refclk to save clock grid power
+        CME_PUTSCOM(PPM_CGCR_CLR, core, BIT64(3));
+
+        // Assert PCB Fence
+        CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(25));
+
+        PK_TRACE("SE2.q");
+        // Update Stop History: In Core Stop Level 2
+        // Check if STOP level 2 reaches the target for both or one core
+        entry_ongoing =
+            target_level == STOP_LEVEL_2 ? STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+        CME_STOP_UPDATE_HISTORY(core,
+                                STOP_CORE_IS_GATED,
+                                entry_ongoing,
+                                target_level,
+                                STOP_LEVEL_2,
+                                STOP_REQ_DISABLE,
+                                STOP_ACT_ENABLE);
+
+        // If both cores targeting different levels
+        // deeper core should have at least deeper stop level than 2
+        // but only need to modify deeper core history if another one was done
+        if (deeper_core && !entry_ongoing)
+        {
+            CME_STOP_UPDATE_HISTORY(deeper_core,
+                                    STOP_CORE_IS_GATED,
+                                    STOP_TRANS_ENTRY,
+                                    deeper_level,
+                                    STOP_LEVEL_2,
+                                    STOP_REQ_DISABLE,
+                                    STOP_ACT_DISABLE);
+            // from now on, proceed with only deeper core
+            core          = deeper_core;
+            target_level  = deeper_level;
+            deeper_level  = 0;
+            deeper_core   = 0;
+            entry_ongoing = 1;
+        }
+
+        PK_TRACE("SE2: Clock Sync Dropped");
+
+        //===========================
+        MARK_TAG(SE_STOP2_DONE, core)
+        //===========================
+
+        //===========================
+        MARK_TRAP(SE_IS0_BEGIN)
+        //===========================
+        // todo: open wakeup interrupt window if we are not done
+        // todo: open catchup interrupt window if we are not done
+        //===================
+        MARK_TRAP(SE_IS0_END)
+        //===================
+
+        // If we are done at STOP level 2
+        if (!entry_ongoing)
+        {
+            break;
+        }
+
+        PK_TRACE("SE2+:core[%d],deeper_core[%d],\
+                  target_level[%d],deeper_level[%d]",
+                 core, deeper_core, target_level, deeper_level);
+
+        //--------------------------------------------------------------------------
+        // STOP LEVEL 3
+        //--------------------------------------------------------------------------
+
+        if (target_level == 3)
+        {
+
+            //==========================
+            MARK_TAG(SE_CORE_VMIN, core)
+            //==========================
+
+            PK_TRACE("SE3.a");
+            // Enable IVRM if not already
+
+            PK_TRACE("SE3.b");
+            // Drop to Vmin
+
+            PK_TRACE("SE3.c");
+            // Update Stop History: In Core Stop Level 3
+            CME_STOP_UPDATE_HISTORY(core,
+                                    STOP_CORE_IS_GATED,
+                                    STOP_TRANS_COMPLETE,
+                                    target_level,
+                                    STOP_LEVEL_3,
+                                    STOP_REQ_DISABLE,
+                                    STOP_ACT_ENABLE);
+
+            // If both cores targeting different levels
+            // deeper core should have at least deeper stop level than 3
+            // only need to modify deeper core history if another one was done
+            if (deeper_core)
+            {
+                CME_STOP_UPDATE_HISTORY(deeper_core,
+                                        STOP_CORE_IS_GATED,
+                                        STOP_TRANS_ENTRY,
+                                        deeper_level,
+                                        STOP_LEVEL_2,
+                                        STOP_REQ_DISABLE,
+                                        STOP_ACT_ENABLE);
+                // from now on, proceed with only deeper core
+                core          = deeper_core;
+                target_level  = deeper_level;
+                deeper_level  = 0;
+                deeper_core   = 0;
+                entry_ongoing = 1;
+            }
+            else
+            {
+                entry_ongoing = 0;
+            }
+
+            //===========================
+            MARK_TAG(SE_STOP3_DONE, core)
+            //===========================
+
+            // If we are done at STOP level 3
+            if (!entry_ongoing)
+            {
+                break;
+            }
+        }
+
+        //--------------------------------------------------------------------------
+        // STOP LEVEL 4
+        //--------------------------------------------------------------------------
+
+        // Skip Power off completely only if single or both core target STOP L9
+        // that is target_level == 9 and deeper_core == 0
+        if (target_level != 9 || (deeper_core && deeper_level != 9))
+        {
+
+            //===============================
+            MARK_TAG(SE_POWER_OFF_CORE, core)
+            //===============================
+
+            // We may skip one of the cores in deeper_core case
+            if (deeper_core)
+            {
+                if (deeper_level == 9)
+                {
+                    core = deeper_core ^ CME_MASK_BC;
+                }
+
+                if (target_level == 9)
+                {
+                    core = deeper_core;
+                }
+            }
+
+#if !STOP_PRIME
+            // Assert Cores Electrical Fences
+            PK_TRACE("SE4.a");
+            CME_PUTSCOM(CPPM_NC0INDIR_OR, core, BIT64(26));
+
+            // Make sure we are not forcing PFET for VDD off
+            // vdd_pfet_force_state == 00 (Nop)
+            PK_TRACE("SE4.b");
+            CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+
+            if (scom_data & BITS64(0, 2))
+            {
+                return CME_STOP_ENTRY_VDD_PFET_NOT_IDLE;
+            }
+
+            // Prepare PFET Controls
+            // vdd_pfet_val/sel_override     = 0 (disbaled)
+            // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
+            PK_TRACE("SE4.c");
+            CME_PUTSCOM(PPM_PFCS_CLR, core, BIT64(4) | BIT64(5) | BIT64(8));
+
+            // Power Off Core VDD
+            // vdd_pfet_force_state = 01 (Force Voff)
+            PK_TRACE("SE4.d");
+            CME_PUTSCOM(PPM_PFCS_OR, core, BIT64(1));
+
+            // Poll for power gate sequencer state: 0x8 (FSM Idle)
+            PK_TRACE("SE4.e");
+
+            do
+            {
+                CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+            }
+            while(!(scom_data & BIT64(42)));
+
+#if !EPM_P9_TUNING
+            // Optional: Poll for vdd_pg_sel being: 0x8
+            PK_TRACE("SE4.f");
+
+            do
+            {
+                CME_GETSCOM(PPM_PFCS, core, CME_SCOM_AND, scom_data);
+            }
+            while(!(scom_data & BIT64(46)));
+
+#endif
+
+            // Turn Off Force Voff
+            // vdd_pfet_force_state = 00 (Nop)
+            PK_TRACE("SE4.g");
+            CME_PUTSCOM(PPM_PFCS_CLR, core, BITS64(0, 2));
+#endif
+
+            PK_TRACE("SE4.h");
+            // Update Stop History: In Core Stop Level 4
+            // Check if STOP level 4 reaches the target for both or one core
+            entry_ongoing =
+                target_level == STOP_LEVEL_4 ? STOP_TRANS_COMPLETE :
+                STOP_TRANS_ENTRY;
+            CME_STOP_UPDATE_HISTORY(core,
+                                    STOP_CORE_IS_GATED,
+                                    entry_ongoing,
+                                    target_level,
+                                    STOP_LEVEL_4,
+                                    STOP_REQ_DISABLE,
+                                    STOP_ACT_ENABLE);
+
+            // restore back to both cores in deeper_core case
+            if (deeper_core)
+            {
+                core = CME_MASK_BC;
+            }
+
+            // If both cores targeting different levels
+            // deeper core should have at least deeper stop level than 2
+            // only need to modify deeper core history if another one was done
+            if (deeper_core && !entry_ongoing)
+            {
+                CME_STOP_UPDATE_HISTORY(deeper_core,
+                                        STOP_CORE_IS_GATED,
+                                        STOP_TRANS_ENTRY,
+                                        deeper_level,
+                                        STOP_LEVEL_4,
+                                        STOP_REQ_DISABLE,
+                                        STOP_ACT_DISABLE);
+                // from now on, proceed with only deeper core
+                core          = deeper_core;
+                target_level  = deeper_level;
+                deeper_level  = 0;
+                deeper_core   = 0;
+                entry_ongoing = 1;
+            }
+
+            PK_TRACE("SE4: Core Powered Off");
+
+            //===========================
+            MARK_TAG(SE_STOP4_DONE, core)
+            //===========================
+
+            //===========================
+            MARK_TRAP(SE_IS1_BEGIN)
+            //===========================
+            // todo: open wakeup interrupt window if we are not done
+            //===================
+            MARK_TRAP(SE_IS1_END)
+            //===================
+
+            // If we are done at STOP level 4
+            if (!entry_ongoing)
+            {
+                break;
+            }
+
+            PK_TRACE("SE4+:core[%d],deeper_core[%d],\
+                      target_level[%d],deeper_level[%d]",
+                     core, deeper_core, target_level, deeper_level);
+        }
+
+        //--------------------------------------------------------------------------
+        // STOP LEVEL 5 (preparsion of STOP LEVEL 8 and above)
+        //--------------------------------------------------------------------------
+
+        if ((G_cme_stop_record.pm_state_c0 >= STOP_LEVEL_8) &&
+            (G_cme_stop_record.pm_state_c1 >= STOP_LEVEL_8))
+        {
+
+            //=========================
+            MARK_TAG(SE_PURGE_L2, core)
+            //=========================
+
+            // Assert L2+NCU Purges(chtm purge will be done in SGPE), and NCU tlbie quiesce
+            PK_TRACE("SE5.1a");
+            out32(CME_LCL_SICR_OR, BIT32(18) | BIT32(21) | BIT32(22));
+
+            //===========================
+            MARK_TRAP(SE_IS2_BEGIN)
+            //===========================
+
+            // todo: open wakeup window to abort purge
+
+            // Poll for Purged Done
+            PK_TRACE("SE5.1b");
+
+            while((in32(CME_LCL_EISR) & BITS32(22, 2)) != BITS32(22, 2));
+
+            // todo: close wakeup window
+
+            //===================
+            MARK_TRAP(SE_IS2_END)
+            //===================
+
+            // Deassert L2+NCU Purges
+            PK_TRACE("SE5.1c");
+            out32(CME_LCL_SICR_CLR, BIT32(18) | BIT32(22));
+
+            // Raise L2-L3 quiesce???
+
+            PK_TRACE("SE5.1: L2/NCU/CHTM Purged");
+
+            //=========================
+            MARK_TRAP(SE_L2_PURGE_DONE)
+            //=========================
+        }
+
+
+        // Recheck status in case L2 purge is aborted by wakeup
+        if ((G_cme_stop_record.pm_state_c0 <= STOP_LEVEL_4) &&
+            (G_cme_stop_record.pm_state_c1 <= STOP_LEVEL_4))
+        {
+            entry_ongoing = 0;
+            break;
+        }
+
+        //=============================
+        MARK_TAG(SE_SGPE_HANDOFF, core)
+        //=============================
+
+        // Update Stop History: In Core Stop Level 5
+        PK_TRACE("SE5.2a");
+        CME_STOP_UPDATE_HISTORY(core,
+                                STOP_CORE_IS_GATED,
+                                STOP_TRANS_CORE_PORTION,
+                                target_level,
+                                STOP_LEVEL_5,
+                                STOP_REQ_DISABLE,
+                                STOP_ACT_ENABLE);
+
+        // Send PCB Interrupt per core
+        PK_TRACE("SE5.2b");
+        pig.fields.req_intr_type = 2; //0b010: STOP State Change
+
+        if (core & CME_MASK_C0)
+        {
+            pig.fields.req_intr_payload = G_cme_stop_record.pm_state_c0;
+            CME_PUTSCOM(PPM_PIG, CME_MASK_C0, pig.value);
+        }
+
+        if (core & CME_MASK_C1)
+        {
+            pig.fields.req_intr_payload = G_cme_stop_record.pm_state_c1;
+            CME_PUTSCOM(PPM_PIG, CME_MASK_C1, pig.value);
+        }
+
+        // Change PPM Wakeup to STOPGPE
+        PK_TRACE("SE5.2c");
+        CME_PUTSCOM(CPPM_CPMMR_OR, core, BIT64(13));
+
+        PK_TRACE("SE5.2: Handed off to SGPE");
+
+    }
+    while(0);
+
+    //--------------------------------------------------------------------------
+    // END OF STOP ENTRY
+    //--------------------------------------------------------------------------
+
+    // Release PPM Write Protection
+    CME_PUTSCOM(CPPM_CPMMR_CLR, core, BIT64(0));
+
+    // Enable fired Stop and corresponding Wakeup Interrupts
+    // if no handoff to SGPE
+    //this exit is umasked, this entry remains masked
+    //other entry is umasked only if it's waken up
+    if (!entry_ongoing)
+        out32(CME_LCL_EIMR_CLR,
+              ((~core & CME_MASK_BC & G_cme_stop_record.cme_wakenup) << SHIFT32(21)) |
+              (core << SHIFT32(13)) |
+              (core << SHIFT32(15)) |
+              (core << SHIFT32(17)));
+
+    //============================
+    MARK_TRAP(ENDSCOPE_STOP_ENTRY)
+    //============================
+
+    return CME_STOP_SUCCESS;
+}
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
new file mode 100644
index 00000000..0febd155
--- /dev/null
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -0,0 +1,217 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h $ */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2015,2018                                                    */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+///
+/// \file p9_cme_stop.h
+/// \brief header of p9_cme_stop_enter_thread.c and p9_cme_stop_exit.c
+///
+
+#include "pk.h"
+#include "ppe42.h"
+#include "ppe42_scom.h"
+
+#include "ppehw_common.h"
+#include "gpehw_common.h"
+#include "occhw_interrupts.h"
+
+#include "ocb_register_addresses.h"
+#include "cme_register_addresses.h"
+#include "ppm_register_addresses.h"
+#include "cppm_register_addresses.h"
+#include "qppm_register_addresses.h"
+
+#include "ocb_firmware_registers.h"
+#include "cme_firmware_registers.h"
+#include "ppm_firmware_registers.h"
+#include "cppm_firmware_registers.h"
+#include "qppm_firmware_registers.h"
+
+#include "p9_stop_common.h"
+
+#define EQ_SCAN_REGION_TYPE      0x10030005
+#define EQ_CLK_REGION            0x10030006
+#define EQ_CLOCK_STAT_SL         0x10030008
+#define EQ_NET_CTRL0_WAND        0x100F0041
+#define EQ_NET_CTRL0_WOR         0x100F0042
+#define EQ_CPLT_CTRL0_CLEAR      0x10000020
+#define EQ_CPLT_CTRL0_OR         0x10000010
+#define EQ_CPLT_CTRL1_CLEAR      0x10000021
+#define EQ_CPLT_CTRL1_OR         0x10000011
+#define EQ_BIST                  0x100F000B
+#define PM_PURGE_REG             0x10011C13
+#define DRAM_REF_REG             0x10011C0F
+#define EQ_QPPM_EDRAM_CTRL_CLEAR 0x100F01BE
+#define EQ_QPPM_EDRAM_CTRL_OR    0x100F01BF
+#define EQ_QPPM_DPLL_CTRL_CLEAR  0x100F0153
+#define EQ_QPPM_DPLL_CTRL_OR     0x100F0154
+#define EQ_QPPM_DPLL_STAT        0x100F0155
+#define EQ_PPM_CGCR_CLEAR        0x100F0166
+#define EQ_PPM_CGCR_OR           0x100F0167
+#define EQ_QPPM_QCCR             0x100F01BD
+#define EQ_QPPM_QCCR_WCLEAR      0x100F01BE
+#define EQ_QPPM_QCCR_WOR         0x100F01BF
+#define EQ_QPPM_QACCR_SCOM1      0x100F0161
+#define EQ_QPPM_QACCR_SCOM2      0x100F0162
+#define EQ_QPPM_QACSR            0x100F0163
+#define EQ_QPPM_EXCGCR           0x100F0165
+#define EQ_QPPM_EXCGCR_CLR       0x100F0166
+#define EQ_QPPM_EXCGCR_OR        0x100F0167
+#define EQ_PM_LCO_DIS_REG        0x0
+#define EQ_PM_L2_RCMD_DIS_REG    0x0
+
+#define SGPE_STOP_L2_CLOCK_REGION(ex) (ex << SHIFT64(9))
+#define SGPE_STOP_L3_CLOCK_REGION(ex) (ex << SHIFT64(7))
+/// Macro to update STOP History
+#define SGPE_STOP_UPDATE_HISTORY(id,base,gated,trans,req_l,act_l,req_e,act_e) \
+    hist.fields.stop_gated       = gated;                                     \
+    hist.fields.stop_transition  = trans;                                     \
+    hist.fields.req_stop_level   = req_l;                                     \
+    hist.fields.act_stop_level   = act_l;                                     \
+    hist.fields.req_write_enable = req_e;                                     \
+    hist.fields.act_write_enable = act_e;                                     \
+    GPE_PUTSCOM(PPM_SSHSRC, base, id, hist.value);
+
+
+enum SGPE_STOP_RETURN_CODES
+{
+    SGPE_STOP_SUCCESS                 = 0,
+    SGPE_STOP_RECEIVE_WRONG_PM_STATE  = 0x00747301,
+    SGPE_STOP_ENTRY_VDD_PFET_NOT_IDLE = 0x00747302
+};
+
+enum SGPE_STOP_IRQ_SHORT_NAMES
+{
+    IRQ_STOP_TYPE2                    = OCCHW_IRQ_PMC_PCB_INTR_TYPE2_PENDING,
+    IRQ_STOP_TYPE5                    = OCCHW_IRQ_PMC_PCB_INTR_TYPE5_PENDING,
+    IRQ_STOP_TYPE6                    = OCCHW_IRQ_PMC_PCB_INTR_TYPE6_PENDING
+};
+
+enum SGPE_STOP_IRQ_PAYLOAD_MASKS
+{
+    TYPE2_PAYLOAD_STOP_EVENT          = 0xC00,
+    TYPE2_PAYLOAD_STOP_LEVEL          = 0xF
+};
+
+enum SGPE_STOP_EVENT_LEVELS
+{
+    SGPE_EX_BASE_LV                   = 8,
+    SGPE_EQ_BASE_LV                   = 9
+};
+
+enum SGPE_STOP_EVENT_FLAGS
+{
+    SGPE_ENTRY_FLAG                   = 2,
+    SGPE_EXIT_FLAG                    = 1
+};
+
+typedef union sgpe_level
+{
+    uint16_t qlevel;
+    struct
+    {
+        uint8_t x0lv : 8;
+        uint8_t x1lv : 8;
+    } xlevel;
+    struct
+    {
+        uint8_t c0lv : 4;
+        uint8_t c1lv : 4;
+        uint8_t c2lv : 4;
+        uint8_t c3lv : 4;
+    } clevel;
+} sgpe_level_t;
+
+typedef union sgpe_state
+{
+    uint32_t status;
+    struct
+    {
+        uint16_t target : 16;
+        uint16_t actual : 16;
+    } differ;
+    struct
+    {
+        uint8_t  spare0 : 4;
+        uint8_t  q_req  : 4;
+        uint8_t  x0req  : 4;
+        uint8_t  x1req  : 4;
+        uint8_t  spare1 : 4;
+        uint8_t  q_act  : 4;
+        uint8_t  x0act  : 4;
+        uint8_t  x1act  : 4;
+    } detail;
+} sgpe_state_t;
+
+typedef union sgpe_group
+{
+    uint64_t vector[2];
+    struct
+    {
+        uint32_t exit;
+        uint32_t entry;
+        uint32_t xq_in;
+        uint32_t spare;
+    } action;
+    struct
+    {
+        uint32_t c_out;
+        uint32_t c_in;
+        uint16_t x_in;
+        uint16_t q_in;
+        uint32_t spare;
+    } member;
+} sgpe_group_t;
+
+
+/// SGPE Stop Score Board Structure
+typedef struct
+{
+    sgpe_group_t group;
+    sgpe_state_t state[MAX_QUADS];
+    sgpe_level_t level[MAX_QUADS];
+    PkSemaphore  sem[2];
+} SgpeStopRecord;
+
+
+/// SGPE STOP Entry and Exit Prototypes
+void p9_sgpe_stop_pig_type2_handler(void*, PkIrqId);
+void p9_sgpe_stop_enter_thread(void*);
+void p9_sgpe_stop_exit_thread(void*);
+int  p9_sgpe_stop_entry();
+int  p9_sgpe_stop_exit();
+
+int  p9_hcd_cache_poweron(uint8_t);
+int  p9_hcd_cache_chiplet_reset(uint8_t);
+int  p9_hcd_cache_gptr_time_initf(uint8_t);
+int  p9_hcd_cache_dpll_setup(uint8_t);
+int  p9_hcd_cache_chiplet_init(uint8_t);
+int  p9_hcd_cache_repair_initf(uint8_t);
+int  p9_hcd_cache_arrayinit(uint8_t);
+int  p9_hcd_cache_initf(uint8_t);
+int  p9_hcd_cache_startclocks(uint8_t);
+int  p9_hcd_cache_l2_startclocks(uint8_t, uint8_t);
+int  p9_hcd_cache_scominit(uint8_t);
+int  p9_hcd_cache_scomcust(uint8_t);
+int  p9_hcd_cache_ras_runtime_scom(uint8_t);
+int  p9_hcd_cache_occ_runtime_scom(uint8_t);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
new file mode 100644
index 00000000..4e53baa2
--- /dev/null
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -0,0 +1,471 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c $ */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2015,2018                                                    */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+
+#include "p9_sgpe_stop.h"
+#include "p9_sgpe_stop_enter_marks.h"
+
+extern SgpeStopRecord G_sgpe_stop_record;
+
+int
+p9_sgpe_stop_entry()
+{
+    int          rc;
+    uint8_t      ex;
+    uint8_t      qloop;
+    uint8_t      cloop;
+    uint8_t      climit;
+    uint8_t      qentry;
+    uint16_t     xentry;
+    uint64_t     scom_data;
+    uint8_t      entry_ongoing[2] = {0, 0};
+    ppm_sshsrc_t hist;
+
+    //===================================================================
+    MARK_TAG(BEGINSCOPE_STOP_ENTRY, G_sgpe_stop_record.group.member.c_in)
+    //===================================================================
+
+    // ------------------------------------------------------------------------
+    // EX STOP ENTRY
+    // ------------------------------------------------------------------------
+
+    for(xentry = G_sgpe_stop_record.group.member.x_in, qloop = 0;
+        xentry > 0;
+        xentry = xentry << 2, qloop++)
+    {
+        // if this ex is not up to entry, skip
+        if (!(ex = ((xentry & BITS16(0, 2)) >> SHIFT16(1))))
+        {
+            continue;
+        }
+
+        PK_TRACE("q[%d]exmask[%d] starts entry", qloop, ex);
+
+        //-------------------------------------------------------------------------
+        // STOP LEVEL 8
+        //-------------------------------------------------------------------------
+
+        // Update QSSR: stop_entry_ongoing
+        out32(OCB_QSSR_OR, BIT32(qloop + 20));
+
+        // Update History for ongoing stop 8 entry
+        cloop  = (ex & FST_EX_IN_QUAD) ?              0 : CORES_PER_EX;
+        climit = (ex & SND_EX_IN_QUAD) ? CORES_PER_QUAD : CORES_PER_EX;
+
+        for(; cloop < climit; cloop++)
+        {
+            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                     CORE_ADDR_BASE,
+                                     STOP_CORE_IS_GATED,
+                                     STOP_TRANS_ENTRY,
+                                     STOP_LEVEL_8,
+                                     STOP_LEVEL_8,
+                                     STOP_REQ_DISABLE,
+                                     STOP_ACT_DISABLE);
+        }
+
+        //========================
+        MARK_TRAP(SE_STOP_L2_CLKS)
+        //========================
+
+        PK_TRACE("SE8.a");
+        // Disable L2 Snoop(quiesce L2-L3 interface, what about NCU?)
+        GPE_PUTSCOM(EQ_PM_L2_RCMD_DIS_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
+
+        PK_TRACE("SE8.b");
+        // Set all bits to zero prior stop core clocks
+        GPE_PUTSCOM(EQ_SCAN_REGION_TYPE, QUAD_ADDR_BASE, qloop, 0);
+
+        PK_TRACE("SE8.c");
+        // Stop L2 Clocks
+        GPE_PUTSCOM(EQ_CLK_REGION, QUAD_ADDR_BASE, qloop,
+                    (0x900000000000E000 | ((uint64_t)ex << SHIFT64(9))));
+
+        PK_TRACE("SE8.d");
+
+        // Poll for L2 clocks stopped
+        do
+        {
+            GPE_GETSCOM(EQ_CLOCK_STAT_SL, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while(((scom_data >> SHIFT64(9)) & ex) != ex);
+
+        // MF: verify compiler generate single rlwmni
+        // MF: delay may be needed for stage latch to propagate thold
+
+        PK_TRACE("SE8: L2 Clock Stopped");
+
+        //========================
+        MARK_TRAP(SE_STOP_L2_GRID)
+        //========================
+
+#if !EPM_P9_TUNING
+        PK_TRACE("SE8.e");
+        // Drop clock sync enable before switch to refclk
+        GPE_PUTSCOM(EQ_QPPM_EXCGCR_CLR, QUAD_ADDR_BASE, qloop,
+                    (ex << SHIFT64(37)));
+
+        PK_TRACE("SE8.f");
+
+        // Poll for clock sync done to drop
+        do
+        {
+            GPE_GETSCOM(QPPM_QACSR, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while((~scom_data >> SHIFT64(37)) & ex != ex);
+
+#endif
+
+        PK_TRACE("SE8.g");
+        // Switch glsmux to refclk to save clock grid power
+        GPE_PUTSCOM(EQ_QPPM_EXCGCR_CLR, QUAD_ADDR_BASE, qloop,
+                    (ex << SHIFT64(35)));
+
+        PK_TRACE("SE8.h");
+
+        if (ex & FST_EX_IN_QUAD)
+        {
+            cloop = 0;
+            G_sgpe_stop_record.state[qloop].detail.x0act = STOP_LEVEL_8;
+            entry_ongoing[0] =
+                G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
+                STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+        }
+        else
+        {
+            cloop = CORES_PER_EX;
+        }
+
+        if (ex & SND_EX_IN_QUAD)
+        {
+            climit = CORES_PER_QUAD;
+            G_sgpe_stop_record.state[qloop].detail.x1act = STOP_LEVEL_8;
+            entry_ongoing[1] =
+                G_sgpe_stop_record.state[qloop].detail.x0req == STOP_LEVEL_8 ?
+                STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+        }
+        else
+        {
+            climit = CORES_PER_EX;
+        }
+
+        for(; cloop < climit; cloop++)
+        {
+            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                     CORE_ADDR_BASE,
+                                     STOP_CORE_IS_GATED,
+                                     entry_ongoing[cloop >> 1],
+                                     STOP_LEVEL_8,
+                                     STOP_LEVEL_8,
+                                     STOP_REQ_DISABLE,
+                                     STOP_ACT_ENABLE);
+        }
+
+        // Update QSSR: l2_stopped, drop stop_entry_ongoing
+        out32(OCB_QSSR_OR, (ex << SHIFT32((qloop << 1) + 1)) | BIT32(qloop + 20));
+
+        //=================================================
+        MARK_TAG(SE_STOP8_DONE, ((ex << 6) | (1 << qloop)))
+        //=================================================
+
+    };
+
+    // ------------------------------------------------------------------------
+    // QUAD STOP ENTRY
+    // ------------------------------------------------------------------------
+
+    for(qentry = G_sgpe_stop_record.group.member.q_in, qloop = 0;
+        qentry > 0;
+        qentry = qentry << 1, qloop++)
+    {
+        // if this quad is not up to entry, skip
+        if (!(qentry & BIT16(0)))
+        {
+            continue;
+        }
+
+        PK_TRACE("q[%d] starts entry", qloop);
+
+        // Update QSSR: stop_entry_ongoing
+        out32(OCB_QSSR_OR, BIT32(qloop + 20));
+
+        // Update STOP History
+        for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
+        {
+            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                     CORE_ADDR_BASE,
+                                     STOP_CORE_IS_GATED,
+                                     STOP_TRANS_ENTRY,
+                                     G_sgpe_stop_record.state[qloop].detail.q_req,
+                                     STOP_LEVEL_9,
+                                     STOP_REQ_ENABLE,
+                                     STOP_ACT_DISABLE);
+        }
+
+        // ------------------------------------------------------------------------
+        // STOP LEVEL 9
+        //-------------------------------------------------------------------------
+
+        //=================================
+        MARK_TAG(SE_PURGE_L3, (1 << qloop))
+        //=================================
+
+        // Assert chtm purges
+        GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE | CME_ADDR_OFFSET_EX0,
+                    qloop, BIT64(24) | BIT64(25));
+        GPE_PUTSCOM(CME_SCOM_SICR_OR, QUAD_ADDR_BASE | CME_ADDR_OFFSET_EX1,
+                    qloop, BIT64(24) | BIT64(25));
+
+        PK_TRACE("SE9.a");
+        // Disable LCO prior to purge
+        GPE_PUTSCOM(EQ_PM_LCO_DIS_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
+
+        PK_TRACE("SE9.b");
+        // Assert Purge L3
+        GPE_PUTSCOM(PM_PURGE_REG, QUAD_ADDR_BASE, qloop, BIT64(0));
+
+        PK_TRACE("SE9.b");
+
+        // Poll for purge done on the same request bit thus no need to deassert
+        do
+        {
+            GPE_GETSCOM(PM_PURGE_REG, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while(scom_data & BIT64(0));
+
+        //====================
+        MARK_TRAP(SE_PURGE_PB)
+        //====================
+
+        PK_TRACE("SE9.b");
+        // Purge PowerBus
+        GPE_PUTSCOM(EQ_QPPM_QCCR_WOR, QUAD_ADDR_BASE, qloop, BIT64(30));
+
+        PK_TRACE("SE9.b");
+
+        // Poll purge PowerBus done
+        do
+        {
+            GPE_GETSCOM(EQ_QPPM_QCCR, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while(!(scom_data & BIT64(31)));
+
+        //==========================================
+        MARK_TAG(SE_WAIT_PGPE_SUSPEND, (1 << qloop))
+        //==========================================
+        // Poll PGPE Suspend Ack
+
+        //=====================================
+        MARK_TAG(SE_STOP_L3_CLKS, (1 << qloop))
+        //=====================================
+
+        PK_TRACE("SE9.d");
+        // Assert refresh quiesce prior to L3 (refresh domain) stop clk
+        // Edram quiesce is asserted by hardware when l3 thold is asserted in cc
+        GPE_PUTSCOM(DRAM_REF_REG, qloop, QUAD_ADDR_BASE, BIT64(7));
+
+        PK_TRACE("SE9.c");
+        // Raise Cache Logical fence
+        GPE_PUTSCOM(EQ_NET_CTRL0_WOR, QUAD_ADDR_BASE, qloop, BIT64(18));
+
+        // clock off cache chiplet
+        PK_TRACE("SE9.e");
+        // Set all bits to zero prior stop core clocks
+        GPE_PUTSCOM(EQ_SCAN_REGION_TYPE, QUAD_ADDR_BASE, qloop, 0);
+
+        PK_TRACE("SE9.f");
+        // Stop Cache Clocks
+        GPE_PUTSCOM(EQ_CLK_REGION, QUAD_ADDR_BASE, qloop, 0x9FFC00000000E000);
+
+        PK_TRACE("SE9.g");
+
+        // Poll for Cache clocks stopped
+        do
+        {
+            GPE_GETSCOM(EQ_CLOCK_STAT_SL, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while((scom_data & BITS64(4, 10)) != BITS64(4, 10));
+
+        // MF: verify compiler generate single rlwmni
+        // MF: delay may be needed for stage latch to propagate thold
+
+        PK_TRACE("SE9: Cache Clock Stopped");
+
+        PK_TRACE("SE9.h");
+        // Switch glsmux to refclk to save clock grid power
+        GPE_PUTSCOM(PPM_CGCR_CLR, QUAD_ADDR_BASE, qloop, BIT64(3));
+
+        PK_TRACE("SE9.i");
+        // Update Stop History
+        G_sgpe_stop_record.state[qloop].detail.q_act = STOP_LEVEL_9;
+        entry_ongoing[0] =
+            G_sgpe_stop_record.state[qloop].detail.q_req == STOP_LEVEL_9 ?
+            STOP_TRANS_COMPLETE : STOP_TRANS_ENTRY;
+        /*
+        for(cloop=0;cloop<CORES_PER_QUAD;cloop++) {
+            SGPE_STOP_UPDATE_HISTORY(((qloop<<2)+cloop),
+                                     CORE_ADDR_BASE,
+                                     STOP_CORE_IS_GATED,
+                                     entry_ongoing[0],
+                                     STOP_LEVEL_9,
+                                     STOP_LEVEL_9,
+                                     STOP_REQ_DISABLE,
+                                     STOP_ACT_ENABLE);
+        }*/
+        SGPE_STOP_UPDATE_HISTORY(qloop,
+                                 QUAD_ADDR_BASE,
+                                 STOP_CORE_IS_GATED,
+                                 entry_ongoing[0],
+                                 STOP_LEVEL_9,
+                                 STOP_LEVEL_9,
+                                 STOP_REQ_DISABLE,
+                                 STOP_ACT_ENABLE);
+
+        // Update QSSR: quad_stopped
+        out32(OCB_QSSR_OR, BIT32(qloop + 14));
+
+        //===================================
+        MARK_TAG(SE_STOP9_DONE, (1 << qloop))
+        //===================================
+
+        if (!entry_ongoing[0])
+        {
+            // Update QSSR: drop stop_entry_ongoing
+            out32(OCB_QSSR_CLR, BIT32(qloop + 20));
+            continue;
+        }
+
+        //-------------------------------------------------------------------------
+        // STOP LEVEL 11
+        //-------------------------------------------------------------------------
+
+        //========================================
+        MARK_TAG(SE_POWER_OFF_CACHE, (1 << qloop))
+        //========================================
+
+        // Assert Cache Electrical Fence
+        PK_TRACE("SE11.a");
+        GPE_PUTSCOM(EQ_NET_CTRL0_WOR, QUAD_ADDR_BASE, qloop, BIT64(26));
+
+        // L3 edram shutdown
+        PK_TRACE("SE11.b");
+        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
+                    BIT64(3) | BIT64(7));
+        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
+                    BIT64(2) | BIT64(6));
+        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
+                    BIT64(1) | BIT64(5));
+        GPE_PUTSCOM(EQ_QPPM_EDRAM_CTRL_CLEAR, QUAD_ADDR_BASE, qloop,
+                    BIT64(0) | BIT64(4));
+
+        // Make sure we are not forcing PFET for VDD or VCS off
+        // vdd_pfet_force_state == 00 (Nop)
+        // vcs_pfet_force_state == 00 (Nop)
+        PK_TRACE("SE11.c");
+        GPE_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+
+        if (scom_data & BITS64(0, 4))
+        {
+            return SGPE_STOP_ENTRY_VDD_PFET_NOT_IDLE;
+        }
+
+        // Prepare PFET Controls
+        // vdd_pfet_val/sel_override     = 0 (disbaled)
+        // vcs_pfet_val/sel_override     = 0 (disbaled)
+        // vdd_pfet_regulation_finger_en = 0 (controled by FSM)
+        PK_TRACE("SE11.d");
+        GPE_PUTSCOM(PPM_PFCS_CLR, QUAD_ADDR_BASE, qloop, BITS64(4, 4) | BIT64(8));
+
+        // Power Off Core VDD
+        // vdd_pfet_force_state = 01 (Force Voff)
+        // vcs_pfet_force_state = 01 (Force Voff)
+        PK_TRACE("SE11.e");
+        GPE_PUTSCOM(PPM_PFCS_OR, QUAD_ADDR_BASE, qloop, BIT64(1) | BIT64(3));
+
+        // Poll for power gate sequencer state: 0x8 (FSM Idle)
+        PK_TRACE("SE11.f");
+
+        do
+        {
+            GPE_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while(!(scom_data & (BIT64(42) | BIT64(50))));
+
+#if !EPM_P9_TUNING
+        // Optional: Poll for vdd_pg_sel/vcs_pg_sel being: 0x8
+        PK_TRACE("SE11.g");
+
+        do
+        {
+            CME_GETSCOM(PPM_PFCS, QUAD_ADDR_BASE, qloop, scom_data);
+        }
+        while(!(scom_data & BIT64(46) | BIT64(54)));
+
+#endif
+
+        // Turn Off Force Voff
+        // vdd_pfet_force_state = 00 (Nop)
+        PK_TRACE("SE11.h");
+        GPE_PUTSCOM(PPM_PFCS_CLR, QUAD_ADDR_BASE, qloop, BITS64(0, 4));
+
+        PK_TRACE("SE11.i");
+        G_sgpe_stop_record.state[qloop].detail.q_act = STOP_LEVEL_11;
+
+        for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
+        {
+            SGPE_STOP_UPDATE_HISTORY(((qloop << 2) + cloop),
+                                     CORE_ADDR_BASE,
+                                     STOP_CORE_IS_GATED,
+                                     STOP_TRANS_COMPLETE,
+                                     STOP_LEVEL_11,
+                                     STOP_LEVEL_11,
+                                     STOP_REQ_DISABLE,
+                                     STOP_ACT_ENABLE);
+        }
+
+        SGPE_STOP_UPDATE_HISTORY(qloop,
+                                 QUAD_ADDR_BASE,
+                                 STOP_CORE_IS_GATED,
+                                 STOP_TRANS_COMPLETE,
+                                 STOP_LEVEL_11,
+                                 STOP_LEVEL_11,
+                                 STOP_REQ_DISABLE,
+                                 STOP_ACT_ENABLE);
+
+        // Update QSSR: drop stop_entry_ongoing
+        out32(OCB_QSSR_CLR, BIT32(qloop + 20));
+
+        //====================================
+        MARK_TAG(SE_STOP11_DONE, (1 << qloop))
+        //====================================
+    };
+
+    // Enable Type2 Interrupt
+    out32(OCB_OIMR1_CLR, BIT32(15));
+
+    //============================
+    MARK_TRAP(ENDSCOPE_STOP_ENTRY)
+    //============================
+    return SGPE_STOP_SUCCESS;
+}