Cache/Core: Istep4 procedure changes for model 9038 and above

Change-Id: I997537274ea9538330d9fb1ce240de793d90feec
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/23019
Tested-by: Jenkins Server
Tested-by: PPE CI
Tested-by: Hostboot CI
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/23021
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>

5 files changed, 142 insertions, 83 deletions

diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
index 93c59e21..46ab8b2c 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
@@ -36,6 +36,7 @@
 // Includes
 //-----------------------------------------------------------------------------
 
+#include <p9_quad_scom_addresses.H>
 #include <p9_misc_scom_addresses.H>
 #include <p9_perv_sbe_cmn.H>
 #include <p9_hcd_common.H>
@@ -94,6 +95,10 @@ p9_hcd_cache_arrayinit(
         l_region_scan0 |= p9hcd::SCAN0_REGION_EX1_L2_L3_REFR;
     }
 
+    /// @todo add DD1 attribute control
+    FAPI_DBG("DD1 only: set sdis_n(flushing LCBES condition workaround");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CONF0_OR, MASK_SET(34)));
+
 #ifndef P9_HCD_STOP_SKIP_ARRAYINIT
 
     FAPI_DBG("Arrayinit all regions except vital/DPLL");
@@ -124,6 +129,10 @@ p9_hcd_cache_arrayinit(
 
 #endif
 
+    /// @todo add DD1 attribute control
+    FAPI_DBG("DD1 only: reset sdis_n(flushing LCBES condition workaround");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CONF0_CLEAR, MASK_SET(34)));
+
 fapi_try_exit:
 
     FAPI_INF("<<p9_hcd_cache_arrayinit");
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
index b262d614..db4141e7 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
@@ -50,6 +50,27 @@
 // Constant Definitions
 //------------------------------------------------------------------------------
 
+enum P9_HCD_CACHE_CHIPLET_RESET_CONSTANTS
+{
+    // (1)PCB_EP_RESET
+    // (2)CLK_ASYNC_RESET
+    // (3)PLL_TEST_EN
+    // (4)PLLRST
+    // (5)PLLBYP
+    // (11)EDIS
+    // (12)VITL_MPW1
+    // (13)VITL_MPW2
+    // (14)VITL_MPW3
+    // (16)VITL_THOLD
+    // (18)FENCE_EN
+    // (22)FUNC_CLKSEL
+    // (25)PCB_FENCE
+    // (26)LVLTRANS_FENCE
+    Q_NET_CTRL0_INIT_VECTOR = (BITS64(1, 5) | BITS64(11, 4) | BIT64(16) |
+                               BIT64(18) | BIT64(22) | BITS64(25, 2)),
+    CACHE_GLSMUX_RESET_DELAY_REF_CYCLES = 40
+};
+
 //------------------------------------------------------------------------------
 // Procedure: Cache Chiplet Reset
 //------------------------------------------------------------------------------
@@ -64,7 +85,6 @@ p9_hcd_cache_chiplet_reset(
     uint64_t                                    l_l2gmux_input       = 0;
     uint64_t                                    l_l2gmux_reset       = 0;
     uint8_t                                     l_attr_chip_unit_pos = 0;
-    fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> FAPI_SYSTEM;
     fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_chip =
         i_target.getParent<fapi2::TARGET_TYPE_PROC_CHIP>();
     fapi2::Target<fapi2::TARGET_TYPE_PERV>      l_perv =
@@ -115,12 +135,12 @@ p9_hcd_cache_chiplet_reset(
                          MASK_SET(2)));
     }
 
-    FAPI_DBG("Init heartbeat hang counter");
-    l_data64.flush<0>().setBit<2>();
-    FAPI_TRY(putScom(i_target, EQ_HANG_PULSE_6_REG, l_data64));
+    /// @todo needs to revisit this sim workaround
+    FAPI_DBG("Init heartbeat hang counter via HANG_PULSE_6[2]");
+    FAPI_TRY(putScom(i_target, EQ_HANG_PULSE_6_REG, MASK_SET(2)));
 
-    FAPI_DBG("Init NET_CTRL0[1-5,11-14,18,22,26],step needed for hotplug");
-    l_data64 = p9hcd::Q_NET_CTRL0_INIT_VECTOR;
+    FAPI_DBG("Init NET_CTRL0[1-5,11-14,16,18,22,25,26],step needed for hotplug");
+    l_data64 = Q_NET_CTRL0_INIT_VECTOR;
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0, l_data64));
 
     FAPI_DBG("Assert progdly/DCC bypass,L2 DCC reset via NET_CTRL1[1,2,23,24]");
@@ -139,12 +159,18 @@ p9_hcd_cache_chiplet_reset(
     FAPI_DBG("Drop vital thold via NET_CTRL0[16]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(16)));
 
+    /// @todo optional setup sector buffer strength, pulse mode and pulsed mode enable
+
     FAPI_DBG("Drop cache glsmux reset via PPM_CGCR[0]");
     FAPI_TRY(putScom(i_target, EQ_PPM_CGCR, MASK_SET(3)));
 
     FAPI_DBG("Drop L2 glsmux reset via QPPM_EXCGCR[32:33]");
     FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_CLEAR, l_l2gmux_reset));
 
+    FAPI_TRY(fapi2::delay(
+                 CACHE_GLSMUX_RESET_DELAY_REF_CYCLES * p9hcd::CLK_PERIOD_10NS,
+                 CACHE_GLSMUX_RESET_DELAY_REF_CYCLES * p9hcd::SIM_CYCLE_200UD));
+
     FAPI_DBG("Assert chiplet enable via NET_CTRL0[0]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(0)));
 
@@ -191,8 +217,11 @@ p9_hcd_cache_chiplet_reset(
 #endif
 
     /// @todo scan_with_setpulse_module(L3 DCC)
-    ///FAPI_DBG("Drop L3 DCC bypass via NET_CTRL1[1]");
-    ///FAPI_TRY(putScom(i_target, EQ_NET_CTRL1_WAND, MASK_UNSET(1)));
+    //FAPI_DBG("Drop L3 DCC bypass via NET_CTRL1[1]");
+    //FAPI_TRY(putScom(i_target, EQ_NET_CTRL1_WAND, MASK_UNSET(1)));
+    /// @todo add VDM_ENABLE attribute control
+    //FAPI_DBG("Assert vdm enable via CPPM_VDMCR[0]");
+    //FAPI_TRY(putScom(i_target, EQ_PPM_VDMCR_OR, MASK_SET(0)));
 
 fapi_try_exit:
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
index 7234ee0d..daa56c18 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
@@ -112,30 +112,32 @@ p9_hcd_cache_dpll_setup(
     FAPI_DBG("Clear all bits prior start DPLL clock via SCAN_REGION_TYPE");
     FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
 
-    FAPI_DBG("Start DPLL clock(arrays+nsl clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_DPLL_THOLD_NSL_ARY;
+    FAPI_DBG("Start DPLL clock via CLK_REGION");
+    l_data64 = (p9hcd::CLK_START_CMD   |
+                p9hcd::CLK_REGION_DPLL |
+                p9hcd::CLK_THOLD_ALL);
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
-    /// @todo parameterize delay
-    FAPI_TRY(fapi2::delay(0, 1000000));
-
-    FAPI_DBG("Start DPLL clock(sl+refresh clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_DPLL_THOLD_ALL;
-    FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
-
-    FAPI_DBG("Poll for DPLL clock running via CLOCK_STAT_SL[14]");
+    FAPI_DBG("Poll for DPLL clock running via CPLT_STAT0[8]");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_DPLL_CLK_START_TIMEOUT_IN_MS;
 
     do
     {
-        FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+        FAPI_TRY(getScom(i_target, EQ_CPLT_STAT0, l_data64));
     }
-    while(((l_data64 & BIT64(14)) != 0) && ((--l_timeout) != 0));
+    while((l_data64.getBit<8>() != 1) && ((--l_timeout) != 0));
 
     FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_DPLLCLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
+                fapi2::PMPROC_DPLLCLKSTART_TIMEOUT().set_EQCPLTSTAT(l_data64),
                 "DPLL Clock Start Timeout");
+
+    FAPI_DBG("Check DPLL clock running via CLOCK_STAT_SL[14]");
+    FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+
+    FAPI_ASSERT((l_data64.getBit<14>() == 0),
+                fapi2::PMPROC_DPLLCLKSTART_FAILED().set_EQCLKSTAT(l_data64),
+                "DPLL Clock Start Failed");
     FAPI_DBG("DPLL clock running now");
 
     // This is necessary to ensure that the DPLL is in Mode 1(ff_bypass = 1)
@@ -179,40 +181,6 @@ p9_hcd_cache_dpll_setup(
     FAPI_DBG("Drop skew/duty cycle adjust func_clksel via NET_CTRL0[22]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(22)));
 
-    // ----------------
-    // Start ANEP clock
-    /// @todo remove this step starting with Chip Drop DT1
-    // ----------------
-
-    FAPI_DBG("Clear all bits prior start ANEP clock via SCAN_REGION_TYPE");
-    FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
-
-    FAPI_DBG("Start ANEP clock(arrays+nsl clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_ANEP_THOLD_NSL_ARY;
-    FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
-
-    /// @todo parameterize delay
-    FAPI_TRY(fapi2::delay(0, 1000000));
-
-    FAPI_DBG("Start ANEP clock(sl+refresh clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_ANEP_THOLD_ALL;
-    FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
-
-    FAPI_DBG("Poll for ANEP clock running via CLOCK_STAT_SL[10]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                CACHE_ANEP_CLK_START_TIMEOUT_IN_MS;
-
-    do
-    {
-        FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
-    }
-    while(((l_data64 & BIT64(10)) != 0) && ((--l_timeout) != 0));
-
-    FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_ANEPCLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
-                "ANEP Clock Start Timeout");
-    FAPI_DBG("ANEP clock running now");
-
     FAPI_DBG("Drop skew adjust reset via NET_CTRL0[2]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(2)));
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
index 7b53d74c..d5fea0ca 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
@@ -37,10 +37,13 @@
 //------------------------------------------------------------------------------
 // Includes
 //------------------------------------------------------------------------------
-#include "p9_hcd_cache_scominit.H"
+
+#include <p9_quad_scom_addresses.H>
+#include <p9_hcd_common.H>
 #include <p9_l2_scom.H>
 #include <p9_l3_scom.H>
 #include <p9_ncu_scom.H>
+#include "p9_hcd_cache_scominit.H"
 
 //------------------------------------------------------------------------------
 // Constant Definitions
@@ -56,8 +59,7 @@ p9_hcd_cache_scominit(
     const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
 {
     FAPI_INF(">>p9_hcd_cache_scominit");
-
-    /// @todo actual scom init content will be required for L3
+    fapi2::buffer<uint64_t> l_data64;
 
     fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> FAPI_SYSTEM;
     auto l_ex_targets = i_target.getChildren<fapi2::TARGET_TYPE_EX>();
@@ -93,10 +95,16 @@ p9_hcd_cache_scominit(
         }
     }
 
+    /// @todo set the sample pulse count (bit 6:9)
+    /// enable the appropriate loops
+    /// (needs investigation with the Perv team on the EC wiring).
+    FAPI_DBG("Enable DTS sampling via THERM_MODE_REG[5]");
+    FAPI_TRY(getScom(i_target, EQ_THERM_MODE_REG, l_data64));
+    FAPI_TRY(putScom(i_target, EQ_THERM_MODE_REG, DATA_SET(5)));
+
 fapi_try_exit:
 
     FAPI_INF("<<p9_hcd_cache_scominit");
-
     return fapi2::current_err;
 }
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
index 644cb31d..69233cea 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
@@ -66,8 +66,9 @@
 
 enum P9_HCD_CACHE_STARTCLOCKS_CONSTANTS
 {
-    CACHE_CLK_SYNC_TIMEOUT_IN_MS = 1,
-    CACHE_CLK_START_TIMEOUT_IN_MS = 1
+    CACHE_CLK_SYNC_TIMEOUT_IN_MS        = 1,
+    CACHE_CLK_START_TIMEOUT_IN_MS       = 1,
+    CACHE_CLK_ALIGN_DELAY_CACHE_CYCLES  = 255
 };
 
 //------------------------------------------------------------------------------
@@ -81,24 +82,33 @@ p9_hcd_cache_startclocks(
     FAPI_INF(">>p9_hcd_cache_startclocks");
     fapi2::buffer<uint64_t>                     l_qcsr;
     fapi2::buffer<uint64_t>                     l_data64;
-    uint64_t                                    l_l2sync_clock;
     uint64_t                                    l_region_clock;
+    uint64_t                                    l_l2sync_clock;
+    uint64_t                                    l_l2pscom_mask;
+    uint64_t                                    l_l3pscom_mask;
     uint32_t                                    l_timeout;
     uint32_t                                    l_attr_system_id     = 0;
     uint8_t                                     l_attr_group_id      = 0;
     uint8_t                                     l_attr_chip_id       = 0;
     uint8_t                                     l_attr_chip_unit_pos = 0;
+    uint8_t                                     l_attr_system_ipl_phase;
+    uint32_t                                    l_attr_pg;
     fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_chip =
         i_target.getParent<fapi2::TARGET_TYPE_PROC_CHIP>();
     fapi2::Target<fapi2::TARGET_TYPE_PERV>      l_perv =
         i_target.getParent<fapi2::TARGET_TYPE_PERV>();
+    fapi2::Target<fapi2::TARGET_TYPE_SYSTEM>    l_sys;
 
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_SYSTEM_IPL_PHASE,      l_sys,
+                           l_attr_system_ipl_phase));
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_GROUP_ID,  l_chip,
                            l_attr_group_id));
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_CHIP_ID,   l_chip,
                            l_attr_chip_id));
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_SYSTEM_ID, l_chip,
                            l_attr_system_id));
+    FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PG,                    l_perv,
+                           l_attr_pg));
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS,         l_perv,
                            l_attr_chip_unit_pos));
     l_attr_chip_unit_pos = l_attr_chip_unit_pos - p9hcd::PERV_TO_EQ_POS_OFFSET;
@@ -110,9 +120,17 @@ p9_hcd_cache_startclocks(
     // -----------------------------
     // Prepare to start cache clocks
     // -----------------------------
-
-    l_region_clock = p9hcd::CLK_REGION_ALL_BUT_EX_DPLL;
+    // QCCR[0/4] EDRAM_ENABLE_DC
+    // QCCR[1/5] EDRAM_VWL_ENABLE_DC
+    // QCCR[2/6] L3_EX0/1_EDRAM_VROW_VBLH_ENABLE_DC
+    // QCCR[3/7] EDRAM_VPP_ENABLE_DC
+    // 0x0 -> 0x8 -> 0xC -> 0xE -> 0xF to turn on edram
+    // stagger EDRAM turn-on per EX (not both at same time)
+
+    l_region_clock = p9hcd::CLK_REGION_ALL_BUT_EX_ANEP_DPLL;
     l_l2sync_clock = 0;
+    l_l2pscom_mask = 0;
+    l_l3pscom_mask = 0;
 
     if (l_qcsr & BIT64(l_attr_chip_unit_pos << 1))
     {
@@ -128,6 +146,11 @@ p9_hcd_cache_startclocks(
         FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(3)));
         FAPI_TRY(fapi2::delay(1000, 200));
     }
+    else
+    {
+        l_l2pscom_mask |= (BIT64(2) | BIT64(10));
+        l_l3pscom_mask |= (BIT64(4) | BIT64(6) | BIT64(8));
+    }
 
     if (l_qcsr & BIT64((l_attr_chip_unit_pos << 1) + 1))
     {
@@ -143,6 +166,14 @@ p9_hcd_cache_startclocks(
         FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(7)));
         FAPI_TRY(fapi2::delay(1000, 200));
     }
+    else
+    {
+        l_l2pscom_mask |= (BIT64(3) | BIT64(11));
+        l_l3pscom_mask |= (BIT64(5) | BIT64(7) | BIT64(9));
+    }
+
+    FAPI_DBG("Assert cache EX1 ID bit2 via CPLT_CTRL0[6]");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_OR, MASK_SET(6)));
 
     FAPI_DBG("Set inop_align/wait/wait_cycles via OPCG_ALIGN[0-3,12-19,52-63]");
     FAPI_TRY(getScom(i_target, EQ_OPCG_ALIGN, l_data64));
@@ -151,8 +182,9 @@ p9_hcd_cache_startclocks(
     insertFromRight<52, 12>(0x10);
     FAPI_TRY(putScom(i_target, EQ_OPCG_ALIGN, l_data64));
 
-    FAPI_DBG("Drop partial good fences via CPLT_CTRL1[3-14]");
-    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, l_region_clock));
+    FAPI_DBG("Drop partial good fences via CPLT_CTRL1[4,5,6/7,8/9,10,11,12/13]");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR,
+                     (l_region_clock | p9hcd::CLK_REGION_ANEP)));
 
     FAPI_DBG("Drop vital fence via CPLT_CTRL1[3]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, MASK_SET(3)));
@@ -202,6 +234,12 @@ p9_hcd_cache_startclocks(
     FAPI_TRY(putScom(i_target, EQ_SYNC_CONFIG, DATA_SET(7)));
     FAPI_TRY(putScom(i_target, EQ_SYNC_CONFIG, DATA_UNSET(7)));
 
+    FAPI_TRY(fapi2::delay(
+                 CACHE_CLK_ALIGN_DELAY_CACHE_CYCLES * p9hcd::CLK_PERIOD_CORE2CACHE *
+                 p9hcd::CLK_PERIOD_250PS / 1000,
+                 CACHE_CLK_ALIGN_DELAY_CACHE_CYCLES * p9hcd::CLK_PERIOD_CORE2CACHE *
+                 p9hcd::SIM_CYCLE_4U4D));
+
     FAPI_DBG("Poll for cache chiplet aligned");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_CLK_START_TIMEOUT_IN_MS;
@@ -221,8 +259,6 @@ p9_hcd_cache_startclocks(
     FAPI_DBG("Drop force_align via CPLT_CTRL0[3]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(3)));
 
-    FAPI_TRY(fapi2::delay(0, 900));
-
     // -------------------------------
     // Start cache clocks
     // -------------------------------
@@ -230,38 +266,43 @@ p9_hcd_cache_startclocks(
     FAPI_DBG("Clear all bits prior start cache clocks via SCAN_REGION_TYPE");
     FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
 
-    FAPI_DBG("Start cache clocks(arrays+nsl clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_NONE_THOLD_NSL_ARY | l_region_clock;
+    FAPI_DBG("Start cache clocks(all but anep+dpll) via CLK_REGION");
+    l_data64 = (p9hcd::CLK_START_CMD |
+                l_region_clock       |
+                p9hcd::CLK_THOLD_ALL);
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
-    /// @todo parameterize delay
-    FAPI_TRY(fapi2::delay(0, 1000000));
-
-    FAPI_DBG("Start cache clocks(sl+refresh clock region) via CLK_REGION");
-    l_data64 = p9hcd::CLK_START_REGION_NONE_THOLD_ALL | l_region_clock;
-    FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
-
-    FAPI_DBG("Poll for cache clocks running via CLOCK_STAT_SL[4-14]");
+    FAPI_DBG("Poll for cache clocks running via CPLT_STAT0[8]");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_CLK_START_TIMEOUT_IN_MS;
 
     do
     {
-        FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+        FAPI_TRY(getScom(i_target, EQ_CPLT_STAT0, l_data64));
     }
-    while(((l_data64 & l_region_clock) != 0) && ((--l_timeout) != 0));
+    while((l_data64.getBit<8>() != 1) && ((--l_timeout) != 0));
 
     FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CACHECLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
+                fapi2::PMPROC_CACHECLKSTART_TIMEOUT().set_EQCPLTSTAT(l_data64),
                 "Cache Clock Start Timeout");
+
+    FAPI_DBG("Check cache clocks running");
+    FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+
+    FAPI_ASSERT(((l_data64 & l_region_clock) == 0),
+                fapi2::PMPROC_CACHECLKSTART_FAILED().set_EQCLKSTAT(l_data64),
+                "Cache Clock Start Failed");
     FAPI_DBG("Cache clocks running now");
 
     // -------------------------------
     // Cleaning up
     // -------------------------------
 
-    FAPI_DBG("Drop chiplet fence via NET_CTRL0[18]");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(18)));
+    if (((~l_attr_pg) & BITS32(4, 11)) && l_attr_system_ipl_phase != 4)
+    {
+        FAPI_DBG("Drop chiplet fence via NET_CTRL0[18]");
+        FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(18)));
+    }
 
     /// @todo ignore xstop checkstop in sim, review for lab
     /*
@@ -275,6 +316,10 @@ p9_hcd_cache_startclocks(
     FAPI_DBG("Drop flushmode_inhibit via CPLT_CTRL0[2]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(2)));
 
+    FAPI_DBG("Drop partial good and assert partial bad L2/L3 pscom masks");
+    l_data64 = (l_l2pscom_mask | l_l3pscom_mask);
+    FAPI_TRY(putScom(i_target, EQ_RING_FENCE_MASK_LATCH_REG, l_data64));
+
 fapi_try_exit:
 
     FAPI_INF("<<p9_hcd_cache_startclocks");