HB: fix HB core boot resulting cme boot

Change-Id: I18f7c24dc84536126a90a251ae770b0498b5d3d0
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/31138
Dev-Ready: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Reviewed-by: AMIT KUMAR <akumar3@us.ibm.com>
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Dev-Ready: Michael S. Floyd <mfloyd@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

9 files changed, 165 insertions, 125 deletions

diff --git a/import/chips/p9/common/pmlib/include/ppehw_common.h b/import/chips/p9/common/pmlib/include/ppehw_common.h
index df802bde..7d065026 100644
--- a/import/chips/p9/common/pmlib/include/ppehw_common.h
+++ b/import/chips/p9/common/pmlib/include/ppehw_common.h
@@ -41,8 +41,8 @@
 /// Create a multi-bit mask of \a n bits starting at bit \a b
 #define BITS64(b, n) ((0xffffffffffffffffull << (64 - (n))) >> (b))
 #define BITS32(b, n) ((0xffffffff            << (32 - (n))) >> (b))
-#define BITS16(b, n) ((0xffff                << (16 - (n))) >> (b))
-#define BITS8(b, n)  ((0xff                  << (8  - (n))) >> (b))
+#define BITS16(b, n) (((0xffff               << (16 - (n))) & 0xffff) >> (b))
+#define BITS8(b, n)  (((0xff                 << (8  - (n))) & 0xff) >> (b))
 
 /// Create a single bit mask at bit \a b
 #define BIT64(b) BITS64((b), 1)
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
index 519444e2..d5e530f4 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_chiplet_reset.c
@@ -119,6 +119,9 @@ p9_hcd_cache_chiplet_reset(uint32_t quad, uint32_t ex)
     PK_TRACE("Drop PCB fence via NET_CTRL0[25]");
     GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(25));
 
+    PK_TRACE("Assert sram_enable via NET_CTRL0[23]");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, quad), BIT64(23));
+
     PK_TRACE("Set scan ratio to 1:1 in bypass mode via OPCG_ALIGN[47-51]");
     GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_OPCG_ALIGN, quad), scom_data);
     scom_data &= ~BITS64(47, 5);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
index 681a2e2d..b2b7c6c6 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
@@ -32,7 +32,8 @@ int
 p9_hcd_cache_scominit(uint32_t quad, uint32_t ex)
 {
     int rc = SGPE_STOP_SUCCESS;
-    uint64_t   scom_data;
+    int exloop, excount = 0;
+    data64_t   scom_data;
     ocb_qcsr_t qcsr;
 
     PK_TRACE("Set L3_LCO_TARGET_ID/VICTIMS via EX_L3_MODE_REG1[2-5,6-21]");
@@ -44,28 +45,60 @@ p9_hcd_cache_scominit(uint32_t quad, uint32_t ex)
     }
     while (qcsr.fields.change_in_progress);
 
+    for (exloop = 0; exloop < 12; exloop++)
+    {
+        if (qcsr.value & BIT32(exloop))
+        {
+            excount++;
+        }
+    }
+
     if (ex & FST_EX_IN_QUAD)
     {
-        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data);
-        scom_data |= (quad << SHIFT32((5 - 1)));
-        scom_data |= ((qcsr.value & BITS32(0, 12)) >> 6);
-        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data);
+        PK_TRACE("Setup L3-LCO on ex0 via EX_L3_MODE_REG1[0,2-5,6-21]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
+        scom_data.words.upper |= (quad << SHIFT32((5 - 1)));
+        scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
+
+        if (excount > 1)
+        {
+            scom_data.words.upper |= BIT32(0);
+        }
+
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
+
+        PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex0 via EX_L3_MODE_REG0[9]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
+        scom_data.words.upper |= BIT32(9);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
     }
 
     if (ex & SND_EX_IN_QUAD)
     {
-        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data);
-        scom_data |= ((quad << SHIFT32((5 - 1))) + 1);
-        scom_data |= ((qcsr.value & BITS32(0, 12)) >> 6);
-        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data);
+        PK_TRACE("Setup L3-LCO on ex1 via EX_L3_MODE_REG1[0,2-5,6-21]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
+        scom_data.words.upper |= ((quad << SHIFT32((5 - 1))) + 1);
+        scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
+
+        if (excount > 1)
+        {
+            scom_data.words.upper |= BIT32(0);
+        }
+
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
+
+        PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex1 via EX_L3_MODE_REG0[9]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
+        scom_data.words.upper |= BIT32(9);
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
     }
 
     PK_TRACE("Enable DTS sampling via THERM_MODE_REG[5]");
-    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_THERM_MODE_REG, quad), scom_data);
-    scom_data |= BIT64(5);
+    GPE_GETSCOM(GPE_SCOM_ADDR_QUAD(EQ_THERM_MODE_REG, quad), scom_data.value);
+    scom_data.value |= BIT64(5);
     /// @todo set the sample pulse count (bit 6:9)
     /// enable the appropriate loops (needs investigation with the Perv team on the EC wiring).
-    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_THERM_MODE_REG, quad), scom_data);
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_THERM_MODE_REG, quad), scom_data.value);
 
     return rc;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
index 82d7795a..e28d1396 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_sgpe_boot_cme.c
@@ -50,8 +50,8 @@ enum
     HARD_RESET_PPE              =   0x6000000000000000ll,   // Hard Reset PPE
     SBASE_FIRST_BLOCK_COPY      =   0,  // corresponds to address 0xFFFF8000
     INACTIVE_CORE               =   0x00,
-    EVEN_CORE_ACTIVE            =   0x01,
-    ODD_CORE_ACTIVE             =   0x02,
+    EVEN_CORE_ACTIVE            =   0x02,
+    ODD_CORE_ACTIVE             =   0x01,
     BOTH_CORE_ACTIVE            =   0x03,
     BCEBAR0                     =   0,
     BCEBAR1                     =   1,
@@ -157,22 +157,8 @@ BootErrorCode_t boot_cme( uint16_t i_bootCme )
                 continue;
             }
 
-            // Update CME state to Flag register
             quadId = l_cmeIndex >> 1;
             activeCmeList[l_cmeIndex] = coreCnt;
-            l_scomAddr = SGPE_SCOM_ADDR( SCOM_ADDR_CME_FLAGS_CLR,  // @bug:  was SCOM_ADDR_CME_FLAGS
-                                         quadId,
-                                         (l_cmeIndex % 2) ); // @bug: was l_cmeIndex which goes from 0 to 11 (0xE)
-            //    and this should be giving a 0 or 1 value within
-            //    a quad.
-            PPE_PUTSCOM( l_scomAddr, WRITE_CLR_ALL ); // clear all bits first.
-
-            //Writing core status as found in CCSR
-            l_dataReg = activeCmeList[l_cmeIndex];
-            l_scomAddr = SGPE_SCOM_ADDR( SCOM_ADDR_CME_FLAGS,
-                                         quadId,
-                                         (l_cmeIndex % 2) );
-            PPE_PUTSCOM( l_scomAddr, (l_dataReg << CME_FLAG_SHIFT_POS) )
 
             // core is available and CME can attempt to boot it.
             // From SGPE platform, let us first get control of Block copy engine.
@@ -416,27 +402,7 @@ BootErrorCode_t boot_cme( uint16_t i_bootCme )
                     }
 
                     l_cmeRdyCnt++;
-
-                    if( l_dataReg & CME_STOP_READY )
-                    {
-                        if( EVEN_CORE_ACTIVE == activeCmeList[l_cmeIndex] ||
-                            BOTH_CORE_ACTIVE == activeCmeList[l_cmeIndex] )
-                        {
-                            l_scomAddr = GPE_SCOM_ADDR_CORE( SCOM_ADDR_CORE_CPMMR_CLR, (l_cmeIndex << 1 ));
-                            l_dataReg = WKUP_NOTIFY_SELECT;
-                            PPE_PUTSCOM( l_scomAddr, l_dataReg );
-                        }
-
-                        if( ODD_CORE_ACTIVE == activeCmeList[l_cmeIndex] ||
-                            BOTH_CORE_ACTIVE == activeCmeList[l_cmeIndex] )
-                        {
-                            l_scomAddr = GPE_SCOM_ADDR_CORE( SCOM_ADDR_CORE_CPMMR_CLR, ((l_cmeIndex << 1) + 1 ));
-                            l_dataReg = WKUP_NOTIFY_SELECT;
-                            PPE_PUTSCOM( l_scomAddr, l_dataReg );
-                        }
-
-                        cmeReadyList = cmeReadyList | l_cmeActiveBit;
-                    }
+                    cmeReadyList = cmeReadyList | l_cmeActiveBit;
                 }//end for
 
                 PPE_WAIT_CORE_CYCLES(cme_rdy_loop, 256);
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
index 82932c62..cc785dd8 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop.h
@@ -95,6 +95,7 @@ extern "C" {
 #define EQ_QPPM_QCCR_WOR         0x100F01BF
 
 #define EX_NCU_STATUS_REG        0x1001100F
+#define EX_L3_MODE_REG0          0x1001182B
 #define EX_L3_MODE_REG1          0x1001180A
 #define EX_DRAM_REF_REG          0x1001180F
 #define EX_PM_PURGE_REG          0x10011813
@@ -155,11 +156,12 @@ enum SGPE_STOP_EVENT_LEVELS
 
 enum SGPE_STOP_CME_FLAGS
 {
+    CME_EX1_INDICATOR                 = BIT64(26),
+    CME_SIBLING_FUNCTIONAL            = BIT64(27),
     CME_CORE0_ENTRY_FIRST             = BIT64(28),
     CME_CORE1_ENTRY_FIRST             = BIT64(29),
     CME_CORE0_ENABLE                  = BIT64(30),
-    CME_CORE1_ENABLE                  = BIT64(31),
-    CME_EX1_INDICATOR                 = BIT64(26)
+    CME_CORE1_ENABLE                  = BIT64(31)
 };
 
 enum SGPE_STOP_PSCOM_MASK
@@ -184,6 +186,17 @@ enum SGPE_STOP_VECTOR_INDEX
 #define RESTORE_RING_BITS(mask, ring, save) ring = (((ring) & (~mask)) | (save));
 #endif
 
+/// 64bits data
+typedef union
+{
+    uint64_t value;
+    struct
+    {
+        uint32_t upper;
+        uint32_t lower;
+    } words;
+} data64_t;
+
 typedef struct
 {
     // requested stop state calculated from core stop levels
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
index 58bfaf67..62c9aef8 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_entry.c
@@ -665,6 +665,9 @@ p9_sgpe_stop_entry()
         PK_TRACE("Assert cache chiplet fence via NET_CTRL0[18]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(18));
 
+        PK_TRACE("Switch glsmux to refclk to save clock grid power via CGCR[3]");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), 0);
+
         PK_TRACE("Clear SCAN_REGION prior to stop cache clocks");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_SCAN_REGION_TYPE, qloop), 0);
 
@@ -694,12 +697,6 @@ p9_sgpe_stop_entry()
             pk_halt();
         }
 
-        // MF: verify compiler generate single rlwmni
-        // MF: delay may be needed for stage latch to propagate thold
-
-        PK_TRACE("Switch glsmux to refclk to save clock grid power via CGCR[3]");
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_PPM_CGCR, qloop), 0);
-
         PK_TRACE("Assert vital fence via CPLT_CTRL1[3]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_CPLT_CTRL1_OR, qloop), BIT64(3));
 
@@ -712,6 +709,9 @@ p9_sgpe_stop_entry()
                      ((uint64_t)ex << SHIFT64(9)) |
                      ((uint64_t)ex << SHIFT64(13))));
 
+        PK_TRACE("Drop CME_INTERPPM_DPLL_ENABLE after DPLL is stopped via QPMMR[26]");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, qloop), BIT64(20) | BIT64(22) | BIT64(24) | BIT64(26));
+
         /// @todo add VDM_ENABLE attribute control
         PK_TRACE("Drop vdm enable via CPPM_VDMCR[0]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(PPM_VDMCR_CLR, qloop), BIT64(0));
@@ -787,6 +787,9 @@ p9_sgpe_stop_entry()
         PK_TRACE("Assert electrical fence via NET_CTRL0[26]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(26));
 
+        PK_TRACE("Drop sram_enable via NET_CTRL0[23]");
+        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, qloop), ~BIT64(23));
+
         PK_TRACE("Assert vital thold via NET_CTRL0[16]");
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WOR, qloop), BIT64(16));
 
@@ -864,9 +867,6 @@ p9_sgpe_stop_entry()
 
 #endif
 
-        PK_TRACE("Drop CME_INTERPPM_DPLL_ENABLE after DPLL is stopped via QPMMR[26]");
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(QPPM_QPMMR_CLR, qloop), BIT64(20) | BIT64(22) | BIT64(24) | BIT64(26));
-
         G_sgpe_stop_record.state[qloop].act_state_q = STOP_LEVEL_11;
 
         for(cloop = 0; cloop < CORES_PER_QUAD; cloop++)
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index dea77fff..89ba8ba9 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -26,7 +26,7 @@
 #include "p9_sgpe_stop.h"
 #include "p9_sgpe_stop_exit_marks.h"
 #include "p9_hcd_sgpe_boot_cme.h"
-//#include "p9_dd1_doorbell_wr.h"
+#include "p9_dd1_doorbell_wr.h"
 
 extern SgpeStopRecord G_sgpe_stop_record;
 
@@ -482,16 +482,10 @@ p9_sgpe_stop_exit()
                 }
 
                 PK_TRACE_INF("SX11.O: Core[%d] DB1 to CME", ((qloop << 2) + cloop));
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEMSG,
-                                               ((qloop << 2) + cloop)), BIT64(0));
-                GPE_PUTSCOM(GPE_SCOM_ADDR_CORE(CPPM_CMEDB1_OR,
-                                               ((qloop << 2) + cloop)), BIT64(7));
-                /*
                 p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEMSG,
-                                               ((qloop << 2) + cloop)), BIT64(0));
+                                                        ((qloop << 2) + cloop)), BIT64(0));
                 p9_dd1_db_unicast_wr(GPE_SCOM_ADDR_CORE(CPPM_CMEDB1_OR,
-                                               ((qloop << 2) + cloop)), BIT64(7));
-                */
+                                                        ((qloop << 2) + cloop)), BIT64(7));
             }
 
             // Setting up cme_flags
@@ -505,6 +499,11 @@ p9_sgpe_stop_exit()
             {
                 cme_flags = 0;
 
+                if (m_pg & SND_EX_IN_QUAD)
+                {
+                    cme_flags |= CME_SIBLING_FUNCTIONAL;
+                }
+
                 if (ccsr.value & BIT32((qloop << 2)))
                 {
                     cme_flags |= CME_CORE0_ENABLE;
@@ -537,6 +536,11 @@ p9_sgpe_stop_exit()
             {
                 cme_flags = CME_EX1_INDICATOR;
 
+                if (m_pg & FST_EX_IN_QUAD)
+                {
+                    cme_flags |= CME_SIBLING_FUNCTIONAL;
+                }
+
                 if (ccsr.value & BIT32(((qloop << 2) + 2)))
                 {
                     cme_flags |= CME_CORE0_ENABLE;
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
index 287c4bb1..76927ae5 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/pk_app_cfg.h
@@ -34,42 +34,37 @@
 /// \brief Application specific overrides go here.
 ///
 
-// --------------------
+// @todo RTC 161182
 
-#ifndef EPM_P9_TUNING
-    #define EPM_P9_TUNING 0
+#if NIMBUS_DD_LEVEL == 1
+    #define HW386311_DD1_PBIE_RW_PTR_STOP11_FIX            1
+    #define HW388878_DD1_VCS_POWER_ON_IN_CHIPLET_RESET_FIX 1
+    #define FUSED_CORE_MODE_SCAN_FIX                       0
 #endif
 
-#define STOP_PRIME           0
-#define SKIP_L3_PURGE        0
-#define SKIP_L3_PURGE_ABORT  0
-#define SKIP_ARRAYINIT       0
-#define SKIP_SCAN0           0
-#define SKIP_INITF           0
+// --------------------
 
 #if EPM_P9_TUNING
-    // @todo RTC 161182
+    // EPM already consumed hardware fix
+    #undef  HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
     #define HW386311_DD1_PBIE_RW_PTR_STOP11_FIX            0
+
+    // EPM doesnt have this problem
+    #undef  HW388878_DD1_VCS_POWER_ON_IN_CHIPLET_RESET_FIX
     #define HW388878_DD1_VCS_POWER_ON_IN_CHIPLET_RESET_FIX 0
-    #define FUSED_CORE_MODE_SCAN_FIX                       0
 
+    // EPM uses broadside RTX instead of BCE
+    #undef  SKIP_CME_BOOT_STOP11
     #define SKIP_CME_BOOT_STOP11 1
-    #define SKIP_CME_BOOT_IPL_HB 1
-    #define EPM_BROADSIDE_SCAN0  0
-#else
-    #define HW386311_DD1_PBIE_RW_PTR_STOP11_FIX            1
-    #define HW388878_DD1_VCS_POWER_ON_IN_CHIPLET_RESET_FIX 1
-    #define FUSED_CORE_MODE_SCAN_FIX                       0
 
-    #define SKIP_CME_BOOT_STOP11 0
+    #undef  SKIP_CME_BOOT_IPL_HB
     #define SKIP_CME_BOOT_IPL_HB 1
-    #define EPM_BROADSIDE_SCAN0  0
+
+    #define PK_TRACE_BUFFER_WRAP_MARKER 1
 #endif
 
 // --------------------
 
-#define PK_TRACE_LEVEL 2
-
 #if PK_TRACE_LEVEL == 0   /*No TRACEs*/
     #define PK_TRACE_ENABLE        0
     #define PK_KERNEL_TRACE_ENABLE 0
@@ -85,49 +80,20 @@
     #define PK_KERNEL_TRACE_ENABLE 1
 #endif
 
-#if EPM_P9_TUNING
-    #define PK_TRACE_BUFFER_WRAP_MARKER 1
-#endif
-#define PK_TRACE_TIMER_OUTPUT  0
-
-// --------------------
-
-#define SIMICS_TUNING 0
-#define USE_SIMICS_IO 0
-#define DEV_DEBUG     1
-
 // --------------------
 
-// This application will use the external timebase register
-// (comment this line out to use the decrementer as timebase)
-#define APPCFG_USE_EXT_TIMEBASE
-
-
 // If we are using the external timebase then assume
 // a frequency of 37.5Mhz.  Otherwise, the default is to use
-// the decrementer as a timebase and assume a frequency of
-// 600MHz
+// the decrementer as a timebase and assume a frequency of 600MHz
 // In product code, this value will be IPL-time configurable.
 #ifdef APPCFG_USE_EXT_TIMEBASE
     #define PPE_TIMEBASE_HZ 37500000
 #else
     #define PPE_TIMEBASE_HZ 600000000
-#endif /* APPCFG_USE_EXT_TIMEBASE */
+#endif
 
 // --------------------
 
-// GPE3 is the SGPE and is the route owner for now.
-// (Change this to #4 for the 405 when we pull that in.)
-#define OCCHW_IRQ_ROUTE_OWNER  3
-
-/// The Instance ID of the OCC processor that this application is intended to run on
-///// 0-3 -> GPE, 4 -> 405
-#define APPCFG_OCC_INSTANCE_ID 3
-
-/// This application will statically initialize it's external interrupt table
-/// using the table defined in pk_app_irq_table.c.
-#define STATIC_IRQ_TABLE
-
 /// About OCCHW_IRQ_PMC_PCB_INTR_TYPE1_PENDING :
 /// This interrupt is used by the PGPE (GPE2) exclusively. Thus, rather than
 ///   defining the entire OCCHW_IRQ_ROUTES table here, and thus over-riding
@@ -147,4 +113,5 @@
     OCCHW_IRQ_PMC_PCB_INTR_TYPE2_PENDING  OCCHW_IRQ_TYPE_EDGE  OCCHW_IRQ_POLARITY_RISING  OCCHW_IRQ_MASKED \
     OCCHW_IRQ_PMC_PCB_INTR_TYPE3_PENDING  OCCHW_IRQ_TYPE_EDGE  OCCHW_IRQ_POLARITY_RISING  OCCHW_IRQ_MASKED \
     OCCHW_IRQ_PMC_PCB_INTR_TYPE6_PENDING  OCCHW_IRQ_TYPE_EDGE  OCCHW_IRQ_POLARITY_RISING  OCCHW_IRQ_MASKED
+
 #endif /*__PK_APP_CFG_H__*/
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/stop_gpe.mk b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/stop_gpe.mk
index 4d5fb162..2cb64e25 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/stop_gpe.mk
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/stop_gpe.mk
@@ -98,8 +98,60 @@ OBJS+=$(UTILS_OBJECTS)
 $(IMAGE)_TRACE_HASH_PREFIX := $(shell echo $(IMAGE) | md5sum | cut -c1-4 \
 	| xargs -i printf "%d" 0x{})
 
-#Note: Flags are resolved very late - so local variables can't be
-# used to build them
+# Note: Flags are resolved very late - 
+# so local variables can't be used to build them
+
+# Options for PK_TRACE
+
+$(IMAGE)_COMMONFLAGS+= -DPK_TRACE_LEVEL=2
+$(IMAGE)_COMMONFLAGS+= -DPK_TRACE_TIMER_OUTPUT=0
+$(IMAGE)_COMMONFLAGS+= -DDEV_DEBUG=0
+
+# Options for Platforms specific tuning
+
+$(IMAGE)_COMMONFLAGS+= -DNIMBUS_DD_LEVEL=1
+$(IMAGE)_COMMONFLAGS+= -DCUMULUS_DD_LEVEL=0
+
+$(IMAGE)_COMMONFLAGS+= -DEPM_P9_TUNING=0
+$(IMAGE)_COMMONFLAGS+= -DEPM_BROADSIDE_SCAN0=0
+
+$(IMAGE)_COMMONFLAGS+= -DSIMICS_TUNING=0
+$(IMAGE)_COMMONFLAGS+= -DUSE_SIMICS_IO=0
+
+# Options for Functions being skipped
+
+$(IMAGE)_COMMONFLAGS+= -DSTOP_PRIME=0
+
+$(IMAGE)_COMMONFLAGS+= -DSKIP_L3_PURGE=0
+$(IMAGE)_COMMONFLAGS+= -DSKIP_L3_PURGE_ABORT=0
+
+$(IMAGE)_COMMONFLAGS+= -DSKIP_ARRAYINIT=0
+$(IMAGE)_COMMONFLAGS+= -DSKIP_SCAN0=0
+$(IMAGE)_COMMONFLAGS+= -DSKIP_INITF=0
+
+$(IMAGE)_COMMONFLAGS+= -DSKIP_CME_BOOT_STOP11=0
+$(IMAGE)_COMMONFLAGS+= -DSKIP_CME_BOOT_IPL_HB=0
+
+# Options for Kernel support
+
+# The Instance ID of the OCC processor 
+# that this application is intended to run on
+# 0-3 -> GPE, 4 -> 405
+$(IMAGE)_COMMONFLAGS+= -DAPPCFG_OCC_INSTANCE_ID=3
+
+# GPE3 is the SGPE and is the route owner for now.
+# (Change this to #4 for the 405 when we pull that in.)
+$(IMAGE)_COMMONFLAGS+= -DOCCHW_IRQ_ROUTE_OWNER=3
+
+# This application will statically initialize 
+# it's external interrupt table using 
+# the table defined in p9_sgpe_main.c.
+$(IMAGE)_COMMONFLAGS+= -DSTATIC_IRQ_TABLE
+
+# This application will use the external timebase register
+# (comment this line out to use the decrementer as timebase)
+$(IMAGE)_COMMONFLAGS+= -DAPPCFG_USE_EXT_TIMEBASE
+
 $(IMAGE)_COMMONFLAGS+= -DPK_TIMER_SUPPORT=1
 $(IMAGE)_COMMONFLAGS+= -DPK_THREAD_SUPPORT=1
 $(IMAGE)_COMMONFLAGS+= -DPK_TRACE_SUPPORT=1
@@ -107,6 +159,8 @@ $(IMAGE)_COMMONFLAGS+= -DUSE_PK_APP_CFG_H=1
 $(IMAGE)_COMMONFLAGS+= -D__PK__=1
 $(IMAGE)_COMMONFLAGS+= -D__PPE_PLAT
 $(IMAGE)_COMMONFLAGS+= -D__PPE__
+
+# Options for FAPI2
 $(IMAGE)_COMMONFLAGS+= -DFAPI2_NO_FFDC=1
 ##$(IMAGE)_COMMONFLAGS+= -DFAPI_TRACE_LEVEL_DEF=3
 $($(IMAGE)_TARGET)_CXXFLAGS += -Wno-unused-label