Revert "Hcode: Drop chiplet fence after scominit and scomcust hwp."

This reverts commit 8da1c05a2580a96b3cc4a46b8cd25ecce06ee208.

Change-Id: I6c23f3a6dadc1b55a3df8f2d879fe225f227a639
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/36067
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

7 files changed, 162 insertions, 117 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
index 373a5adb..00025661 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_exit.c
@@ -51,12 +51,10 @@ p9_cme_stop_exit()
     uint32_t     spwu_stop;
     uint32_t     spwu_wake;
 #endif
-#if !STOP_PRIME
 #if !SKIP_EXIT_CATCHUP
     uint8_t      catchup_level;
     uint32_t     core_catchup;
 #endif
-#endif
 #if HW386841_DD1_PLS_SRR1_DLS_STOP1_FIX
     uint8_t      dsl;
     uint8_t      thread;
@@ -229,6 +227,39 @@ p9_cme_stop_exit()
 
 
 
+    if (deeper_core && target_level == 2)
+    {
+
+        //--------------------------------------------------------------------------
+        PK_TRACE_INF("+++++ +++++ STOP LEVEL 2 EXIT EXPRESS +++++ +++++");
+        //--------------------------------------------------------------------------
+
+        core = CME_MASK_BC - deeper_core;
+
+        PK_TRACE_DBG("Check: Core[%d] Serviced by SX2EX", core);
+
+        //============================
+        MARK_TAG(SX_STARTCLOCKS, core)
+        //============================
+
+        // do this again here for stop2 in addition to chiplet_reset
+        // Note IPL doesnt need to do this twice
+        PK_TRACE("Assert core glitchless mux to DPLL via CGCR[3]");
+        CME_PUTSCOM(C_PPM_CGCR, core, BIT64(3));
+
+        PK_TRACE_INF("SX2.X: Start Core Clock on Core");
+        p9_hcd_core_startclocks(core);
+
+        PK_TRACE("Clear CPPM PECE shadow via PECES");
+        CME_PUTSCOM(CPPM_PECES, core, 0);
+
+        core = deeper_core;
+        deeper_core = 0;
+
+    }
+
+
+
     if (deeper_level >= STOP_LEVEL_4)
     {
 
@@ -533,19 +564,7 @@ p9_cme_stop_exit()
     PK_TRACE("Clear CPPM PECE shadow via PECES");
     CME_PUTSCOM(CPPM_PECES, core, 0);
 
-    if (target_level == 2)
-    {
-        PK_TRACE("Drop chiplet fence via NC0INDIR[18]");
 
-        if (deeper_core)
-        {
-            CME_PUTSCOM(CPPM_NC0INDIR_CLR, (CME_MASK_BC - deeper_core), BIT64(18));
-        }
-        else
-        {
-            CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(18));
-        }
-    }
 
     if (deeper_level >= STOP_LEVEL_4)
     {
@@ -584,12 +603,11 @@ p9_cme_stop_exit()
             pk_halt();
         }
 
-
-#endif
-
         PK_TRACE_INF("SX4.K: XIP Customized Scoms on Core", core);
         p9_hcd_core_scomcust(core);
 
+#endif
+
         //==============================
         MARK_TAG(SX_RUNTIME_INITS, core)
         //==============================
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_scomcust.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_scomcust.c
index 6b8881d8..1bbf98fe 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_scomcust.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_scomcust.c
@@ -30,8 +30,17 @@
 int
 p9_hcd_core_scomcust(uint32_t core)
 {
-    PK_TRACE("Drop chiplet fence via NC0INDIR[18]");
-    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(18));
+    /*
+        CmeImageHeader_t* pCmeImgHdr  = (CmeImageHeader_t*)(CME_HEADER_IMAGE_OFFSET);
+        CmeScomRestore*   pCmeScomRes = (CmeScomRestore*)(pCmeImgHdr->coreScomRestoreOffset);
+        int i;
 
+        for(i=0; pCmeScomRes->pad; i++, pCmeScomRes += sizeof(CmeScomRestore))
+        {
+            PK_TRACE("scom[%d] addr[%x] data[%016llx]",
+                     i, pCmeScomRes->addr, pCmeScomRes->data);
+            CME_PUTSCOM(pCmeScomRes->addr, core, pCmeScomRes->data);
+        }
+    */
     return 0;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
index 059f3a79..7cb51fc8 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_hcd_core_startclocks.c
@@ -37,6 +37,9 @@ p9_hcd_core_startclocks(uint32_t core)
     cmeHeader_t* pCmeImgHdr = (cmeHeader_t*)(CME_SRAM_BASE + CME_HEADER_IMAGE_OFFSET);
     id_vector = pCmeImgHdr->g_cme_location_id;
 
+    //FAPI_DBG("Set CPLT_CTRL0[AVP_MODE] for cache-contained execution");
+    //FAPI_TRY(putScom(i_target, C_CPLT_CTRL0_OR, MASK_SET(5)));
+
     PK_TRACE("Assert sdis_n(flushing LCBES condition) via CPLT_CONF0[34]");
     CME_PUTSCOM(C_CPLT_CONF0_OR, core, BIT64(34));
 
@@ -137,6 +140,9 @@ p9_hcd_core_startclocks(uint32_t core)
     PK_TRACE("Core clock is now running");
     MARK_TRAP(SX_STARTCLOCKS_DONE)
 
+    PK_TRACE("Drop chiplet fence via NC0INDIR[18]");
+    CME_PUTSCOM(CPPM_NC0INDIR_CLR, core, BIT64(18));
+
     /// @todo RTC166917 Check the Global Checkstop FIR
 
     PK_TRACE("Drop flushmode_inhibit via CPLT_CTRL0[2]");
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scomcust.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scomcust.c
index bcb3f7f9..6e076d18 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scomcust.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scomcust.c
@@ -31,9 +31,6 @@ int
 p9_hcd_cache_scomcust(uint32_t quad, uint32_t m_ex, int is_stop8)
 {
     int rc = SGPE_STOP_SUCCESS;
-
-#if !SKIP_HOMER_ACCESS
-
     int i;
     uint32_t qoffset = 0;
     uint32_t qaddr   = 0;
@@ -93,13 +90,5 @@ p9_hcd_cache_scomcust(uint32_t quad, uint32_t m_ex, int is_stop8)
         GPE_PUTSCOM(qaddr, qdata);
     }
 
-#endif
-
-    if (!is_stop8)
-    {
-        PK_TRACE("Drop chiplet fence via NET_CTRL0[18]");
-        GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(18));
-    }
-
     return rc;
 }
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
index c65d9c8c..3fc48b37 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_scominit.c
@@ -32,13 +32,10 @@ extern SgpeStopRecord G_sgpe_stop_record;
 int
 p9_hcd_cache_scominit(uint32_t quad, uint32_t m_ex, int is_stop8)
 {
-    int           rc = SGPE_STOP_SUCCESS;
-    int           attr_proc_fabric_addr_bar_mode_small_system = 0; // default large
-    int           attr_proc_fabric_dump_mode_chip_is_node     = 0; // default group
-    int           exloop                                      = 0;
-    int           excount                                     = 0;
-    data64_t      scom_data                                   = {0};
-    ocb_qcsr_t    qcsr                                        = {0};
+    int rc = SGPE_STOP_SUCCESS;
+    int attr_proc_fabric_addr_bar_mode_small_system = 0; // default large
+    int attr_proc_fabric_dump_mode_chip_is_node     = 0; // default group
+    data64_t scom_data;
     sgpeHeader_t* pSgpeImgHdr = (sgpeHeader_t*)(SGPE_IMAGE_SRAM_BASE + SGPE_HEADER_IMAGE_OFFSET);
 
     if (pSgpeImgHdr->g_sgpe_reserve_flags & BIT32(16))
@@ -62,46 +59,10 @@ p9_hcd_cache_scominit(uint32_t quad, uint32_t m_ex, int is_stop8)
         scom_data.words.upper |= BITS32(20, 2);// DTS loop1 enable
         GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_THERM_MODE_REG, quad), scom_data.value);
 
-        // read partial good exes for LCO setup below
-        do
-        {
-            qcsr.value = in32(OCB_QCSR);
-        }
-        while (qcsr.fields.change_in_progress);
-
-        for (exloop = 0; exloop < 12; exloop++)
-        {
-            if (qcsr.value & BIT32(exloop))
-            {
-                excount++;
-            }
-        }
-
-        PK_TRACE_DBG("Reading QCSR: %x, excount: %x", qcsr.value, excount);
-
         if (m_ex & FST_EX_IN_QUAD)
         {
             // p9_l3_scom
 
-            PK_TRACE("Setup L3_LCO_TARGET_ID/VICTIMS on ex0 via EX_L3_MODE_REG1[2-5,6-21]");
-            GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
-            scom_data.words.upper &= ~BITS32(2, 20);
-            scom_data.words.upper |= (quad << SHIFT32((5 - 1)));
-            scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
-
-            if (excount > 1)
-            {
-                PK_TRACE("Assert L3_LCO_ENABLE_CFG on ex0 via EX_L3_MODE_REG1[0]");
-                scom_data.words.upper |= BIT32(0);
-            }
-            else
-            {
-                PK_TRACE("Drop L3_LCO_ENABLE_CFG on ex0 via EX_L3_MODE_REG1[0]");
-                scom_data.words.upper &= ~BIT32(0);
-            }
-
-            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
-
             GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
             scom_data.words.upper &= ~(BIT32(1) | BITS32(14, 8) | BIT32(22));
             scom_data.words.upper |= (BIT32(2) | BIT32(17) | BIT32(19)); // set b11 with L2
@@ -111,17 +72,6 @@ p9_hcd_cache_scominit(uint32_t quad, uint32_t m_ex, int is_stop8)
                 scom_data.words.upper |= BIT32(22);
             }
 
-            if (excount == 2)
-            {
-                PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex0 via EX_L3_MODE_REG0[9]");
-                scom_data.words.upper |= BIT32(9);
-            }
-            else
-            {
-                PK_TRACE("Drop L3_DYN_LCO_BLK_DIS_CFG on ex0 via EX_L3_MODE_REG0[9]");
-                scom_data.words.upper &= ~BIT32(9);
-            }
-
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
 
             // p9_ncu_scom
@@ -143,25 +93,6 @@ p9_hcd_cache_scominit(uint32_t quad, uint32_t m_ex, int is_stop8)
         {
             // p9_l3_scom
 
-            PK_TRACE("Setup L3_LCO_TARGET_ID/VICTIMS on ex1 via EX_L3_MODE_REG1[2-5,6-21]");
-            GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
-            scom_data.words.upper &= ~BITS32(2, 20);
-            scom_data.words.upper |= ((quad << SHIFT32((5 - 1))) | BIT32(5));
-            scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
-
-            if (excount > 1)
-            {
-                PK_TRACE("Assert L3_LCO_ENABLE_CFG on ex1 via EX_L3_MODE_REG1[0]");
-                scom_data.words.upper |= BIT32(0);
-            }
-            else
-            {
-                PK_TRACE("Drop L3_LCO_ENABLE_CFG on ex1 via EX_L3_MODE_REG1[0]");
-                scom_data.words.upper &= ~BIT32(0);
-            }
-
-            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
-
             GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
             scom_data.words.upper &= ~(BIT32(1) | BITS32(14, 8) | BIT32(22));
             scom_data.words.upper |= (BIT32(2) | BIT32(17) | BIT32(19)); // set b11 with L2
@@ -171,17 +102,6 @@ p9_hcd_cache_scominit(uint32_t quad, uint32_t m_ex, int is_stop8)
                 scom_data.words.upper |= BIT32(22);
             }
 
-            if (excount == 2)
-            {
-                PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex1 via EX_L3_MODE_REG0[9]");
-                scom_data.words.upper |= BIT32(9);
-            }
-            else
-            {
-                PK_TRACE("Drop L3_DYN_LCO_BLK_DIS_CFG on ex1 via EX_L3_MODE_REG0[9]");
-                scom_data.words.upper &= ~BIT32(9);
-            }
-
             GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
 
             // p9_ncu_scom
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
index 406dad22..73ab9482 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_hcd_cache_startclocks.c
@@ -31,8 +31,10 @@ int
 p9_hcd_cache_startclocks(uint32_t quad, uint32_t ex)
 {
     int        rc = SGPE_STOP_SUCCESS;
+    int        exloop, excount = 0;
     uint32_t   id_vector;
     data64_t   scom_data;
+    ocb_qcsr_t qcsr;
 
     sgpeHeader_t* pSgpeImgHdr = (sgpeHeader_t*)(SGPE_IMAGE_SRAM_BASE + SGPE_HEADER_IMAGE_OFFSET);
     id_vector = pSgpeImgHdr->g_sgpe_location_id;
@@ -190,6 +192,101 @@ p9_hcd_cache_startclocks(uint32_t quad, uint32_t ex)
     // Cleaning up
     // -------------------------------
 
+    // The LCO programming for the istep 16 use-case needs to be done
+    // before the chiplet fence is dropped.
+
+    // read partial good exes
+    do
+    {
+        qcsr.value = in32(OCB_QCSR);
+    }
+    while (qcsr.fields.change_in_progress);
+
+    for (exloop = 0; exloop < 12; exloop++)
+    {
+        if (qcsr.value & BIT32(exloop))
+        {
+            excount++;
+        }
+    }
+
+    PK_TRACE_DBG("Reading QCSR: %x, excount: %x", qcsr.value, excount);
+
+    if (ex & FST_EX_IN_QUAD)
+    {
+        PK_TRACE("Setup L3_LCO_TARGET_ID/VICTIMS on ex0 via EX_L3_MODE_REG1[2-5,6-21]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
+        scom_data.words.upper &= ~BITS32(2, 20);
+        scom_data.words.upper |= (quad << SHIFT32((5 - 1)));
+        scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
+
+        if (excount > 1)
+        {
+            PK_TRACE("Assert L3_LCO_ENABLE_CFG on ex0 via EX_L3_MODE_REG1[0]");
+            scom_data.words.upper |= BIT32(0);
+        }
+        else
+        {
+            PK_TRACE("Drop L3_LCO_ENABLE_CFG on ex0 via EX_L3_MODE_REG1[0]");
+            scom_data.words.upper &= ~BIT32(0);
+        }
+
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 0), scom_data.value);
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
+
+        if (excount == 2)
+        {
+            PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex0 via EX_L3_MODE_REG0[9]");
+            scom_data.words.upper |= BIT32(9);
+            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
+        }
+        else
+        {
+            PK_TRACE("Drop L3_DYN_LCO_BLK_DIS_CFG on ex0 via EX_L3_MODE_REG0[9]");
+            scom_data.words.upper &= ~BIT32(9);
+            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 0), scom_data.value);
+        }
+    }
+
+    if (ex & SND_EX_IN_QUAD)
+    {
+        PK_TRACE("Setup L3_LCO_TARGET_ID/VICTIMS on ex1 via EX_L3_MODE_REG1[2-5,6-21]");
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
+        scom_data.words.upper &= ~BITS32(2, 20);
+        scom_data.words.upper |= ((quad << SHIFT32((5 - 1))) | BIT32(5));
+        scom_data.words.upper |= ((qcsr.value & BITS32(0, 12)) >> 6);
+
+        if (excount > 1)
+        {
+            PK_TRACE("Assert L3_LCO_ENABLE_CFG on ex1 via EX_L3_MODE_REG1[0]");
+            scom_data.words.upper |= BIT32(0);
+        }
+        else
+        {
+            PK_TRACE("Drop L3_LCO_ENABLE_CFG on ex1 via EX_L3_MODE_REG1[0]");
+            scom_data.words.upper &= ~BIT32(0);
+        }
+
+        GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG1, quad, 1), scom_data.value);
+        GPE_GETSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
+
+        if (excount == 2)
+        {
+            PK_TRACE("Assert L3_DYN_LCO_BLK_DIS_CFG on ex1 via EX_L3_MODE_REG0[9]");
+            scom_data.words.upper |= BIT32(9);
+            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
+        }
+        else
+        {
+            PK_TRACE("Drop L3_DYN_LCO_BLK_DIS_CFG on ex1 via EX_L3_MODE_REG0[9]");
+            scom_data.words.upper &= ~BIT32(9);
+            GPE_PUTSCOM(GPE_SCOM_ADDR_EX(EX_L3_MODE_REG0, quad, 1), scom_data.value);
+        }
+    }
+
+    PK_TRACE("Drop chiplet fence via NET_CTRL0[18]");
+    GPE_PUTSCOM(GPE_SCOM_ADDR_QUAD(EQ_NET_CTRL0_WAND, quad), ~BIT64(18));
+
     /// @todo RTC166917 Check the Global Checkstop FIR
 
 #if NIMBUS_DD_LEVEL != 1
diff --git a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
index 1c299d1a..944ffbf4 100644
--- a/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
+++ b/import/chips/p9/procedures/ppe_closed/sgpe/stop_gpe/p9_sgpe_stop_exit.c
@@ -122,14 +122,12 @@ p9_sgpe_stop_exit()
     int          fused_core_mode = 0;
 #endif
     uint64_t     scom_data = 0;
+    uint64_t     cme_flags;
     ppm_sshsrc_t hist;
-#if !STOP_PRIME
     ocb_ccsr_t   ccsr;
-    uint64_t     cme_flags;
 #if HW386311_DD1_PBIE_RW_PTR_STOP11_FIX
     int          spin;
 #endif
-#endif
 #if !SKIP_IPC
     int          rc;
 #endif
@@ -591,9 +589,13 @@ p9_sgpe_stop_exit()
             PK_TRACE_INF("SX8.B: Cache L2 Scominit");
             p9_hcd_cache_scominit(qloop, m_l2, 1);
 
+#if !SKIP_HOMER_ACCESS
+
             PK_TRACE_INF("SX8.C: Cache L2 Scomcust");
             p9_hcd_cache_scomcust(qloop, m_l2, 1);
 
+#endif
+
             // reset ex actual state if ex is exited.
             if (m_l2 & FST_EX_IN_QUAD)
             {
@@ -654,9 +656,13 @@ p9_sgpe_stop_exit()
             PK_TRACE_INF("SX11.M: Cache Scom Init");
             p9_hcd_cache_scominit(qloop, m_pg, 0);
 
+#if !SKIP_HOMER_ACCESS
+
             PK_TRACE_INF("SX11.N: Cache Scom Cust");
             p9_hcd_cache_scomcust(qloop, m_pg, 0);
 
+#endif
+
             //==================================
             MARK_TAG(SX_CME_BOOT, (32 >> qloop))
             //==================================