cache/core/l2_stopclocks updates

patchset 1: skip l2_stopclocks if ex_select is 0
patchset 2: check power state before execute hwp
patchset 3: fix syntax typos from patch 2 update
patchset 4: add chiplet accessibility check
patchset 5: add possible counter to CME PCBMUX
patchset 6: add skipping message on check
patchset 7: change polling timout method
patchset 8: add a missing comma
patchset 9: fix ffdc
patchset 10:roll back cme pcbmux counter until
            check to ensure cme accessibility
patchset 11:rebased
patchset 12:initial checkin of ppe state handling
patchset 13:checkin new clk_ctrl_state procedure
patchset 14:add attribute xml for new procedure
patchset 15:fix calling the p9_common_clk_ctrl_state
patchset 16:Matt rebase
patchset 17:Warning instead of fail with error on check
            delete common C file, include only header

Change-Id: I14c9480ac0931ac7f8b456f193148ceb3b939947
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/28808
Dev-Ready: YUE DU <daviddu@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Matt K. Light <mklight@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/30365
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>

8 files changed, 325 insertions, 207 deletions

diff --git a/src/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_stopclocks.C b/src/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_stopclocks.C
index ab82ac9d..80680dda 100644
--- a/src/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_stopclocks.C
+++ b/src/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_stopclocks.C
@@ -42,6 +42,7 @@
 #include <p9_misc_scom_addresses.H>
 #include <p9_quad_scom_addresses.H>
 #include <p9_hcd_common.H>
+#include <p9_common_clk_ctrl_state.H>
 #include "p9_hcd_l2_stopclocks.H"
 #include "p9_hcd_cache_stopclocks.H"
 
@@ -51,7 +52,8 @@
 
 enum P9_HCD_CACHE_STOPCLOCKS_CONSTANTS
 {
-    CACHE_CLK_STOP_TIMEOUT_IN_MS = 1
+    CACHE_CLK_STOP_POLLING_HW_NS_DELAY     = 10000,
+    CACHE_CLK_STOP_POLLING_SIM_CYCLE_DELAY = 320000
 };
 
 //------------------------------------------------------------------------------
@@ -64,11 +66,13 @@ p9_hcd_cache_stopclocks(
     const p9hcd::P9_HCD_CLK_CTRL_CONSTANTS      i_select_regions,
     const p9hcd::P9_HCD_EX_CTRL_CONSTANTS       i_select_ex)
 {
-    FAPI_INF(">>p9_hcd_cache_stopclocks: regions[%x] ex[%d]",
+    FAPI_INF(">>p9_hcd_cache_stopclocks: regions[%016llx] ex[%d]",
              i_select_regions, i_select_ex);
+    fapi2::ReturnCode                              l_rc;
     fapi2::buffer<uint64_t>                        l_data64;
-    uint32_t                                       l_timeout;
+    fapi2::buffer<uint64_t>                        l_temp64;
     uint64_t                                       l_l3mask_pscom = 0;
+    uint32_t                                       l_loops1ms;
     uint8_t                                        l_attr_chip_unit_pos = 0;
     uint8_t                                        l_attr_vdm_enable;
     const fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> l_sys;
@@ -79,7 +83,7 @@ p9_hcd_cache_stopclocks(
                            l_attr_vdm_enable));
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_perv,
                            l_attr_chip_unit_pos));
-//    l_attr_chip_unit_pos = l_attr_chip_unit_pos - p9hcd::PERV_TO_QUAD_POS_OFFSET;
+//  l_attr_chip_unit_pos = l_attr_chip_unit_pos - p9hcd::PERV_TO_QUAD_POS_OFFSET;
     l_attr_chip_unit_pos = l_attr_chip_unit_pos - 0x10;
 
     if (i_select_regions & p9hcd::CLK_REGION_EX0_L3)
@@ -95,10 +99,37 @@ p9_hcd_cache_stopclocks(
     // -----------------------------
     // Prepare to stop cache clocks
     // -----------------------------
-    /// @todo RTC158181 disable l2 snoop? disable lco? assert refresh quiesce?
 
-    FAPI_DBG("Assert L3 pscom masks via RING_FENCE_MASK_LATCH_REG[4-9]");
-    FAPI_TRY(putScom(i_target, EQ_RING_FENCE_MASK_LATCH_REG, l_l3mask_pscom));
+    FAPI_DBG("Check PM_RESET_STATE_INDICATOR via GPMMR[15]");
+    FAPI_TRY(getScom(i_target, EQ_PPM_GPMMR_SCOM, l_data64));
+
+    if (!l_data64.getBit<15>())
+    {
+        FAPI_DBG("Gracefully turn off power management, if fail, continue anyways");
+        /// @todo suspend_pm()
+    }
+
+    FAPI_DBG("Check cache clock controller status");
+    l_rc = p9_common_clk_ctrl_state<fapi2::TARGET_TYPE_EQ>(i_target);
+
+    if (l_rc)
+    {
+        FAPI_INF("Clock controller of this cache chiplet is inaccessible, return");
+        goto fapi_try_exit;
+    }
+
+    FAPI_DBG("Check PERV clock status for access to CME via CLOCK_STAT[4]");
+    FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+
+    FAPI_DBG("Check PERV fence status for access to CME via CPLT_CTRL1[4]");
+    FAPI_TRY(getScom(i_target, EQ_CPLT_CTRL1, l_temp64));
+
+    if (l_data64.getBit<4>() == 0 && l_temp64.getBit<4>() == 0)
+    {
+        /// @todo RTC158181 disable l2 snoop? disable lco? assert refresh quiesce?
+        FAPI_DBG("Assert L3 pscom masks via RING_FENCE_MASK_LATCH_REG[4-9]");
+        FAPI_TRY(putScom(i_target, EQ_RING_FENCE_MASK_LATCH_REG, l_l3mask_pscom));
+    }
 
     FAPI_DBG("Assert chiplet fence via NET_CTRL0[18]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(18)));
@@ -107,9 +138,10 @@ p9_hcd_cache_stopclocks(
     // Stop L2 clocks
     // -------------------------------
 
-    FAPI_EXEC_HWP(fapi2::current_err,
-                  p9_hcd_l2_stopclocks,
-                  i_target, i_select_ex);
+    if (i_select_ex)
+        FAPI_EXEC_HWP(fapi2::current_err,
+                      p9_hcd_l2_stopclocks,
+                      i_target, i_select_ex);
 
     // -------------------------------
     // Stop cache clocks
@@ -125,28 +157,26 @@ p9_hcd_cache_stopclocks(
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
     FAPI_DBG("Poll for cache clocks stopped via CPLT_STAT0[8]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                CACHE_CLK_STOP_TIMEOUT_IN_MS;
+    l_loops1ms = 1E6 / CACHE_CLK_STOP_POLLING_HW_NS_DELAY;
 
     do
     {
+        fapi2::delay(CACHE_CLK_STOP_POLLING_HW_NS_DELAY,
+                     CACHE_CLK_STOP_POLLING_SIM_CYCLE_DELAY);
+
         FAPI_TRY(getScom(i_target, EQ_CPLT_STAT0, l_data64));
     }
-    while((l_data64.getBit<8>() != 1) && ((--l_timeout) != 0));
+    while((l_data64.getBit<8>() != 1) && ((--l_loops1ms) != 0));
 
-    FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CACHECLKSTOP_TIMEOUT()
-                .set_EQ_TARGET(i_target)
-                .set_EQCPLTSTAT(l_data64),
+    FAPI_ASSERT((l_loops1ms != 0),
+                fapi2::PMPROC_CACHECLKSTOP_TIMEOUT().set_EQCPLTSTAT(l_data64),
                 "Cache Clock Stop Timeout");
 
     FAPI_DBG("Check cache clocks stopped");
     FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
 
     FAPI_ASSERT((((~l_data64) & i_select_regions) == 0),
-                fapi2::PMPROC_CACHECLKSTOP_FAILED()
-                .set_EQ_TARGET(i_target)
-                .set_EQCLKSTAT(l_data64),
+                fapi2::PMPROC_CACHECLKSTOP_FAILED().set_EQCLKSTAT(l_data64),
                 "Cache Clock Stop Failed");
     FAPI_DBG("Cache clocks stopped now");
 
@@ -205,7 +235,7 @@ p9_hcd_cache_stopclocks(
                      BIT64(l_attr_chip_unit_pos + 14)));
 
     FAPI_DBG("Set cache as stopped in STOP history register");
-    FAPI_TRY(putScom(i_target, EQ_PPM_SSHSRC, (BIT64(0) | BIT64(13))));
+    FAPI_TRY(putScom(i_target, EQ_PPM_SSHSRC, BIT64(0)));
 
 fapi_try_exit:
 
diff --git a/src/import/chips/p9/procedures/hwp/cache/p9_hcd_l2_stopclocks.C b/src/import/chips/p9/procedures/hwp/cache/p9_hcd_l2_stopclocks.C
index 301debb7..c4f0657c 100644
--- a/src/import/chips/p9/procedures/hwp/cache/p9_hcd_l2_stopclocks.C
+++ b/src/import/chips/p9/procedures/hwp/cache/p9_hcd_l2_stopclocks.C
@@ -42,6 +42,7 @@
 #include <p9_misc_scom_addresses.H>
 #include <p9_quad_scom_addresses.H>
 #include <p9_hcd_common.H>
+#include <p9_common_clk_ctrl_state.H>
 #include "p9_hcd_l2_stopclocks.H"
 
 //------------------------------------------------------------------------------
@@ -50,8 +51,10 @@
 
 enum P9_HCD_L2_STOPCLOCKS_CONSTANTS
 {
-    L2_CLK_SYNC_TIMEOUT_IN_MS = 1,
-    L2_CLK_STOP_TIMEOUT_IN_MS = 1
+    CACHE_L2_CLK_SYNC_POLLING_HW_NS_DELAY     = 10000,
+    CACHE_L2_CLK_SYNC_POLLING_SIM_CYCLE_DELAY = 320000,
+    CACHE_L2_CLK_STOP_POLLING_HW_NS_DELAY     = 10000,
+    CACHE_L2_CLK_STOP_POLLING_SIM_CYCLE_DELAY = 320000
 };
 
 //------------------------------------------------------------------------------
@@ -64,8 +67,10 @@ p9_hcd_l2_stopclocks(
     const p9hcd::P9_HCD_EX_CTRL_CONSTANTS       i_select_ex)
 {
     FAPI_INF(">>p9_hcd_l2_stopclocks: ex[%d]",  i_select_ex);
+    fapi2::ReturnCode                           l_rc;
     fapi2::buffer<uint64_t>                     l_data64;
-    uint32_t                                    l_timeout;
+    fapi2::buffer<uint64_t>                     l_temp64;
+    uint32_t                                    l_loops1ms;
     uint64_t                                    l_region_clock = 0;
     uint64_t                                    l_l2sync_clock = 0;
     uint64_t                                    l_l2mask_pscom = 0;
@@ -75,7 +80,7 @@ p9_hcd_l2_stopclocks(
 
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_perv,
                            l_attr_chip_unit_pos));
-//    l_attr_chip_unit_pos = l_attr_chip_unit_pos - p9hcd::PERV_TO_QUAD_POS_OFFSET;
+//  l_attr_chip_unit_pos = l_attr_chip_unit_pos - p9hcd::PERV_TO_QUAD_POS_OFFSET;
     l_attr_chip_unit_pos = l_attr_chip_unit_pos - 0x10;
 
     if (i_select_ex & p9hcd::EVEN_EX)
@@ -96,8 +101,35 @@ p9_hcd_l2_stopclocks(
     // Prepare to stop L2 clocks
     // -------------------------
 
-    FAPI_DBG("Assert L2 pscom masks via RING_FENCE_MASK_LATCH_REG[2/3,10/11]");
-    FAPI_TRY(putScom(i_target, EQ_RING_FENCE_MASK_LATCH_REG, l_l2mask_pscom));
+    FAPI_DBG("Check PM_RESET_STATE_INDICATOR via GPMMR[15]");
+    FAPI_TRY(getScom(i_target, EQ_PPM_GPMMR_SCOM, l_data64));
+
+    if (!l_data64.getBit<15>())
+    {
+        FAPI_DBG("Gracefully turn off power management, if fail, continue anyways");
+        /// @todo suspend_pm()
+    }
+
+    FAPI_DBG("Check cache clock controller status");
+    l_rc = p9_common_clk_ctrl_state<fapi2::TARGET_TYPE_EQ>(i_target);
+
+    if (l_rc)
+    {
+        FAPI_INF("Clock controller of this cache chiplet is inaccessible, return");
+        goto fapi_try_exit;
+    }
+
+    FAPI_DBG("Check PERV clock status for access to CME via CLOCK_STAT[4]");
+    FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+
+    FAPI_DBG("Check PERV fence status for access to CME via CPLT_CTRL1[4]");
+    FAPI_TRY(getScom(i_target, EQ_CPLT_CTRL1, l_temp64));
+
+    if (l_data64.getBit<4>() == 0 && l_temp64.getBit<4>() == 0)
+    {
+        FAPI_DBG("Assert L2 pscom masks via RING_FENCE_MASK_LATCH_REG[2/3,10/11]");
+        FAPI_TRY(putScom(i_target, EQ_RING_FENCE_MASK_LATCH_REG, l_l2mask_pscom));
+    }
 
     // -------------------------------
     // Stop L2 clocks
@@ -113,28 +145,26 @@ p9_hcd_l2_stopclocks(
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
     FAPI_DBG("Poll for L2 clocks stopped via CPLT_STAT0[8]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                L2_CLK_STOP_TIMEOUT_IN_MS;
+    l_loops1ms = 1E6 / CACHE_L2_CLK_STOP_POLLING_HW_NS_DELAY;
 
     do
     {
+        fapi2::delay(CACHE_L2_CLK_STOP_POLLING_HW_NS_DELAY,
+                     CACHE_L2_CLK_STOP_POLLING_SIM_CYCLE_DELAY);
+
         FAPI_TRY(getScom(i_target, EQ_CPLT_STAT0, l_data64));
     }
-    while((l_data64.getBit<8>() != 1) && ((--l_timeout) != 0));
+    while((l_data64.getBit<8>() != 1) && ((--l_loops1ms) != 0));
 
-    FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_L2CLKSTOP_TIMEOUT()
-                .set_EQ_TARGET(i_target)
-                .set_EQCPLTSTAT(l_data64),
+    FAPI_ASSERT((l_loops1ms != 0),
+                fapi2::PMPROC_L2CLKSTOP_TIMEOUT().set_EQCPLTSTAT(l_data64),
                 "L2 Clock Stop Timeout");
 
     FAPI_DBG("Check L2 clocks stopped");
     FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
 
     FAPI_ASSERT((((~l_data64) & l_region_clock) == 0),
-                fapi2::PMPROC_L2CLKSTOP_FAILED()
-                .set_EQ_TARGET(i_target)
-                .set_EQCLKSTAT(l_data64),
+                fapi2::PMPROC_L2CLKSTOP_FAILED().set_EQCLKSTAT(l_data64),
                 "L2 Clock Stop Failed");
     FAPI_DBG("L2 clocks stopped now");
 
@@ -146,16 +176,18 @@ p9_hcd_l2_stopclocks(
     FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_CLEAR, l_l2sync_clock));
 
     FAPI_DBG("Poll for L2 clock sync dones to drop via QPPM_QACSR[36,37]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                L2_CLK_SYNC_TIMEOUT_IN_MS;
+    l_loops1ms = 1E6 / CACHE_L2_CLK_STOP_POLLING_HW_NS_DELAY;
 
     do
     {
+        fapi2::delay(CACHE_L2_CLK_STOP_POLLING_HW_NS_DELAY,
+                     CACHE_L2_CLK_STOP_POLLING_SIM_CYCLE_DELAY);
+
         FAPI_TRY(getScom(i_target, EQ_QPPM_QACSR, l_data64));
     }
-    while(((l_data64 & l_l2sync_clock)) && ((--l_timeout) != 0));
+    while(((l_data64 & l_l2sync_clock)) && ((--l_loops1ms) != 0));
 
-    FAPI_ASSERT((l_timeout != 0),
+    FAPI_ASSERT((l_loops1ms != 0),
                 fapi2::PMPROC_CACHECLKSYNCDROP_TIMEOUT().set_EQPPMQACSR(l_data64),
                 "L2 Clock Sync Drop Timeout");
     FAPI_DBG("L2 clock sync dones dropped");
@@ -167,7 +199,6 @@ p9_hcd_l2_stopclocks(
     FAPI_DBG("Assert regional fences via CPLT_CTRL1[8/9]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_OR, l_region_clock));
 
-
     // -------------------------------
     // Update QSSR
     // -------------------------------
@@ -176,7 +207,6 @@ p9_hcd_l2_stopclocks(
     FAPI_TRY(putScom(l_chip, PU_OCB_OCI_QSSR_OR,
                      ((uint64_t)i_select_ex << SHIFT64((l_attr_chip_unit_pos << 1) + 1))));
 
-
 fapi_try_exit:
 
     FAPI_INF("<<p9_hcd_l2_stopclocks");
diff --git a/src/import/chips/p9/procedures/hwp/core/p9_hcd_core_stopclocks.C b/src/import/chips/p9/procedures/hwp/core/p9_hcd_core_stopclocks.C
index 4f586c3d..1aaa7cd5 100644
--- a/src/import/chips/p9/procedures/hwp/core/p9_hcd_core_stopclocks.C
+++ b/src/import/chips/p9/procedures/hwp/core/p9_hcd_core_stopclocks.C
@@ -42,6 +42,7 @@
 #include <p9_misc_scom_addresses.H>
 #include <p9_quad_scom_addresses.H>
 #include <p9_hcd_common.H>
+#include <p9_common_clk_ctrl_state.H>
 #include "p9_hcd_core_stopclocks.H"
 
 //------------------------------------------------------------------------------
@@ -50,8 +51,12 @@
 
 enum P9_HCD_CORE_STOPCLOCKS_CONSTANTS
 {
-    CORE_CLK_SYNC_TIMEOUT_IN_MS       = 1,
-    CORE_CLK_STOP_TIMEOUT_IN_MS       = 1
+    CORE_PCB_MUX_POLLING_HW_NS_DELAY      = 10000,
+    CORE_PCB_MUX_POLLING_SIM_CYCLE_DELAY  = 320000,
+    CORE_CLK_SYNC_POLLING_HW_NS_DELAY     = 10000,
+    CORE_CLK_SYNC_POLLING_SIM_CYCLE_DELAY = 320000,
+    CORE_CLK_STOP_POLLING_HW_NS_DELAY     = 10000,
+    CORE_CLK_STOP_POLLING_SIM_CYCLE_DELAY = 320000
 };
 
 //------------------------------------------------------------------------------
@@ -63,9 +68,11 @@ p9_hcd_core_stopclocks(
     const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target)
 {
     FAPI_INF(">>p9_hcd_core_stopclocks");
+    fapi2::ReturnCode                              l_rc;
     fapi2::buffer<uint64_t>                        l_ccsr;
     fapi2::buffer<uint64_t>                        l_data64;
-    uint32_t                                       l_timeout;
+    fapi2::buffer<uint64_t>                        l_temp64;
+    uint32_t                                       l_loops1ms;
     uint8_t                                        l_attr_chip_unit_pos;
     uint8_t                                        l_attr_vdm_enable;
     const fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> l_sys;
@@ -83,12 +90,59 @@ p9_hcd_core_stopclocks(
     // Prepare to stop core clocks
     // ----------------------------
 
-    FAPI_DBG("Assert Core-L2/CC Quiesces via CME_SCOM_SICR[6,8]/[7,9]");
-    FAPI_TRY(putScom(l_quad,
-                     (l_attr_chip_unit_pos < 2) ?
-                     EX_0_CME_SCOM_SICR_OR : EX_1_CME_SCOM_SICR_OR,
-                     (BIT64(6 + (l_attr_chip_unit_pos % 2)) |
-                      BIT64(8 + (l_attr_chip_unit_pos % 2)))));
+    FAPI_DBG("Check PM_RESET_STATE_INDICATOR via GPMMR[15]");
+    FAPI_TRY(getScom(i_target, C_PPM_GPMMR_SCOM, l_data64));
+
+    if (!l_data64.getBit<15>())
+    {
+        FAPI_DBG("Gracefully turn off power management, continue anyways if fail");
+        /// @todo suspend_pm()
+    }
+
+    FAPI_DBG("Check core clock controller status");
+    l_rc = p9_common_clk_ctrl_state<fapi2::TARGET_TYPE_CORE>(i_target);
+
+    if (l_rc)
+    {
+        FAPI_INF("Clock controller of this core chiplet is inaccessible, return");
+        goto fapi_try_exit;
+    }
+
+    FAPI_DBG("Check cache clock controller status");
+    l_rc = p9_common_clk_ctrl_state<fapi2::TARGET_TYPE_EQ>(l_quad);
+
+    if (l_rc)
+    {
+        FAPI_INF("WARNING: core is enabled while cache is not, continue anyways");
+    }
+    else
+    {
+
+        FAPI_DBG("Check PERV clock status for access to CME via CLOCK_STAT[4]");
+        FAPI_TRY(getScom(l_quad, EQ_CLOCK_STAT_SL, l_data64));
+
+        FAPI_DBG("Check PERV fence status for access to CME via CPLT_CTRL1[4]");
+        FAPI_TRY(getScom(l_quad, EQ_CPLT_CTRL1, l_temp64));
+
+        if (l_data64.getBit<4>() == 0 && l_temp64.getBit<4>() == 0)
+        {
+            //halt cme(poll for halted, if timeout, print warnning keep going).
+
+            FAPI_DBG("Assert Core-L2/CC Quiesces via CME_SCOM_SICR[6,8]/[7,9]");
+            FAPI_TRY(putScom(l_quad,
+                             (l_attr_chip_unit_pos < 2) ?
+                             EX_0_CME_SCOM_SICR_OR : EX_1_CME_SCOM_SICR_OR,
+                             (BIT64(6 + (l_attr_chip_unit_pos % 2)) |
+                              BIT64(8 + (l_attr_chip_unit_pos % 2)))));
+        }
+    }
+
+    FAPI_DBG("Assert pm_mux_disable to get PCB Mux from CME via SLAVE_CONFIG[7]");
+    FAPI_TRY(getScom(i_target, C_SLAVE_CONFIG_REG, l_data64));
+    FAPI_TRY(putScom(i_target, C_SLAVE_CONFIG_REG, DATA_SET(7)));
+
+    FAPI_DBG("Override possible PPM write protection to CME via CPPM_CPMMR[1]");
+    FAPI_TRY(putScom(i_target, C_CPPM_CPMMR_OR, MASK_SET(1)));
 
     FAPI_DBG("Assert chiplet fence via NET_CTRL0[18]");
     FAPI_TRY(putScom(i_target, C_NET_CTRL0_WOR, MASK_SET(18)));
@@ -107,28 +161,26 @@ p9_hcd_core_stopclocks(
     FAPI_TRY(putScom(i_target, C_CLK_REGION, l_data64));
 
     FAPI_DBG("Poll for core clocks stopped via CPLT_STAT0[8]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                CORE_CLK_STOP_TIMEOUT_IN_MS;
+    l_loops1ms = 1E6 / CORE_CLK_STOP_POLLING_HW_NS_DELAY;
 
     do
     {
+        fapi2::delay(CORE_CLK_STOP_POLLING_HW_NS_DELAY,
+                     CORE_CLK_STOP_POLLING_SIM_CYCLE_DELAY);
+
         FAPI_TRY(getScom(i_target, C_CPLT_STAT0, l_data64));
     }
-    while((l_data64.getBit<8>() != 1) && ((--l_timeout) != 0));
+    while((l_data64.getBit<8>() != 1) && ((--l_loops1ms) != 0));
 
-    FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CORECLKSTOP_TIMEOUT()
-                .set_CORE_TARGET(i_target)
-                .set_CORECPLTSTAT(l_data64),
+    FAPI_ASSERT((l_loops1ms != 0),
+                fapi2::PMPROC_CORECLKSTOP_TIMEOUT().set_CORECPLTSTAT(l_data64),
                 "Core Clock Stop Timeout");
 
     FAPI_DBG("Check core clocks stopped via CLOCK_STAT_SL[4-13]");
     FAPI_TRY(getScom(i_target, C_CLOCK_STAT_SL, l_data64));
 
     FAPI_ASSERT((((~l_data64) & p9hcd::CLK_REGION_ALL_BUT_PLL) == 0),
-                fapi2::PMPROC_CORECLKSTOP_FAILED()
-                .set_CORE_TARGET(i_target)
-                .set_CORECLKSTAT(l_data64),
+                fapi2::PMPROC_CORECLKSTOP_FAILED().set_CORECLKSTAT(l_data64),
                 "Core Clock Stop Failed");
     FAPI_DBG("Core clocks stopped now");
 
@@ -140,16 +192,18 @@ p9_hcd_core_stopclocks(
     FAPI_TRY(putScom(i_target, C_CPPM_CACCR_CLEAR, MASK_SET(15)));
 
     FAPI_DBG("Poll for core clock sync done to drop via CPPM_CACSR[13]");
-    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
-                CORE_CLK_STOP_TIMEOUT_IN_MS;
+    l_loops1ms = 1E6 / CORE_CLK_SYNC_POLLING_HW_NS_DELAY;
 
     do
     {
+        fapi2::delay(CORE_CLK_SYNC_POLLING_HW_NS_DELAY,
+                     CORE_CLK_SYNC_POLLING_SIM_CYCLE_DELAY);
+
         FAPI_TRY(getScom(i_target, C_CPPM_CACSR, l_data64));
     }
-    while((l_data64.getBit<13>() == 1) && ((--l_timeout) != 0));
+    while((l_data64.getBit<13>() == 1) && ((--l_loops1ms) != 0));
 
-    FAPI_ASSERT((l_timeout != 0),
+    FAPI_ASSERT((l_loops1ms != 0),
                 fapi2::PMPROC_CORECLKSYNCDROP_TIMEOUT().set_COREPPMCACSR(l_data64),
                 "Core Clock Sync Drop Timeout");
     FAPI_DBG("Core clock sync done dropped");
@@ -186,7 +240,18 @@ p9_hcd_core_stopclocks(
     // -------------------------------
 
     FAPI_DBG("Set core as stopped in STOP history register");
-    FAPI_TRY(putScom(i_target, C_PPM_SSHSRC, (BIT64(0) | BIT64(13))));
+    FAPI_TRY(putScom(i_target, C_PPM_SSHSRC, BIT64(0)));
+
+    // -------------------------------
+    // Clean up
+    // -------------------------------
+
+    FAPI_DBG("Return possible PPM write protection to CME via CPPM_CPMMR[1]");
+    FAPI_TRY(putScom(i_target, C_CPPM_CPMMR_CLEAR, MASK_SET(1)));
+
+    FAPI_DBG("Drop pm_mux_disable to release PCB Mux via SLAVE_CONFIG[7]");
+    FAPI_TRY(getScom(i_target, C_SLAVE_CONFIG_REG, l_data64));
+    FAPI_TRY(putScom(i_target, C_SLAVE_CONFIG_REG, DATA_UNSET(7)));
 
 fapi_try_exit:
 
diff --git a/src/import/chips/p9/procedures/hwp/lib/p9_common_clk_ctrl_state.H b/src/import/chips/p9/procedures/hwp/lib/p9_common_clk_ctrl_state.H
new file mode 100644
index 00000000..1943c328
--- /dev/null
+++ b/src/import/chips/p9/procedures/hwp/lib/p9_common_clk_ctrl_state.H
@@ -0,0 +1,123 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/import/chips/p9/procedures/hwp/lib/p9_common_clk_ctrl_state.H $ */
+/*                                                                        */
+/* OpenPOWER sbe Project                                                  */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2016                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+///
+/// @file  p9_common_clk_ctrl_state.H
+/// @brief Check cache/core clock controller status
+///
+/// Procedure Summary:
+
+// *HWP HWP Owner          : David Du       <daviddu@us.ibm.com>
+// *HWP Backup HWP Owner   : Greg Still     <stillgs@us.ibm.com>
+// *HWP FW Owner           : Sangeetha T S  <sangeet2@in.ibm.com>
+// *HWP Team               : PM
+// *HWP Consumed by        : HB:PERV
+// *HWP Level              : 2
+
+//-----------------------------------------------------------------------------
+// Includes
+//-----------------------------------------------------------------------------
+
+#include <fapi2.H>
+#include <p9_quad_scom_addresses.H>
+#include <p9_hcd_common.H>
+
+//------------------------------------------------------------------------------
+// Constant Definitions
+//------------------------------------------------------------------------------
+
+const uint64_t NET_CTRL0[2] = { C_NET_CTRL0, EQ_NET_CTRL0};
+const uint64_t PPM_PFSNS[2] = { C_PPM_PFSNS, EQ_PPM_PFSNS};
+
+//-----------------------------------------------------------------------------
+// Template Procedure
+//------------------------------------------------------------------------------
+
+template <fapi2::TargetType K>
+fapi2::ReturnCode
+p9_common_clk_ctrl_state(
+    const fapi2::Target<K>& i_target)
+{
+    FAPI_INF(">>p9_common_clk_ctrl_state");
+    fapi2::buffer<uint64_t> l_data64;
+    uint32_t                l_type = 0; // Assumes core
+
+    if((i_target.getType() & fapi2::TARGET_TYPE_EQ))
+    {
+        l_type = 1;
+    }
+
+    FAPI_DBG("Check chiplet accessibility via NET_CTRL0[0,1,16,25,26]");
+    FAPI_TRY(getScom(i_target, NET_CTRL0[l_type], l_data64));
+
+    if (l_data64.getBit<25>())
+    {
+        FAPI_INF("WARNING: Chiplet appears Offline as PCB Fence is up. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+    if (l_data64.getBit<0>() == 0)
+    {
+        FAPI_INF("WARNING: Chiplet Disabled. Assume being Partial Bad. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+    if (l_data64.getBit<1>())
+    {
+        FAPI_INF("WARNING: Chiplet in fixed state as PcbEpReset is on. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+    if (l_data64.getBit<26>())
+    {
+        FAPI_INF("WARNING: Chiplet Elec Fence is up. PCB path blocked. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+    if (l_data64.getBit<16>())
+    {
+        FAPI_INF("WARNING: Chiplet VitalClk off. ClkCtrl inaccessible. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+    FAPI_DBG("Check chiplet power state via PPM_PFSNS[1]");
+    FAPI_TRY(getScom(i_target, PPM_PFSNS[l_type], l_data64));
+
+    if (l_data64.getBit<1>())
+    {
+        FAPI_INF("WARNING: Chiplet out of power while Elec Fence down. Skipping");
+        fapi2::current_err = fapi2::FAPI2_RC_FALSE;
+        goto fapi_try_exit;
+    }
+
+fapi_try_exit:
+
+    FAPI_INF("<<p9_common_clk_ctrl_state");
+    return fapi2::current_err;
+}
diff --git a/src/import/chips/p9/procedures/hwp/lib/p9_hcd_common.H b/src/import/chips/p9/procedures/hwp/lib/p9_hcd_common.H
index 75f1c674..60bedb84 100644
--- a/src/import/chips/p9/procedures/hwp/lib/p9_hcd_common.H
+++ b/src/import/chips/p9/procedures/hwp/lib/p9_hcd_common.H
@@ -43,8 +43,8 @@
 // Create a multi-bit mask of \a n bits starting at bit \a b
 #define BITS64(b, n) ((0xffffffffffffffffull << (64 - (n))) >> (b))
 #define BITS32(b, n) ((0xffffffff            << (32 - (n))) >> (b))
-#define BITS16(b, n) ((0xffff                << (16 - (n))) >> (b))
-#define BITS8(b, n)  ((0xff                  << (8  - (n))) >> (b))
+#define BITS16(b, n) (((0xffff                << (16 - (n))) & 0xffff) >> (b))
+#define BITS8(b, n)  (((0xff                  << (8  - (n))) & 0xff) >> (b))
 
 // Create a single bit mask at bit \a b
 #define BIT64(b) BITS64((b), 1)
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_cache_stopclocks_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_cache_stopclocks_errors.xml
index 592277d9..22d9ffe1 100644
--- a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_cache_stopclocks_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_cache_stopclocks_errors.xml
@@ -38,30 +38,7 @@
     <description>
         cache clock stop failed.
     </description>
-    <ffdc>EQ_TARGET</ffdc>
     <ffdc>EQCLKSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
   <hwpError>
@@ -69,30 +46,7 @@
     <description>
         cache clock stop timed out.
     </description>
-    <ffdc>EQ_TARGET</ffdc>
     <ffdc>EQCPLTSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
 </hwpErrors>
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_core_stopclocks_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_core_stopclocks_errors.xml
index cdf70c33..942abe31 100644
--- a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_core_stopclocks_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_core_stopclocks_errors.xml
@@ -26,6 +26,14 @@
 <hwpErrors>
   <!-- ********************************************************************* -->
   <hwpError>
+    <rc>RC_PMPROC_COREPCBMUX_TIMEOUT</rc>
+    <description>
+        polling for pcb mux grant timed out.
+    </description>
+    <ffdc>CMESISR</ffdc>
+  </hwpError>
+  <!-- ********************************************************************* -->
+  <hwpError>
     <rc>RC_PMPROC_CORECLKSYNCDROP_TIMEOUT</rc>
     <description>
         core clock sync done drop timed out.
@@ -38,30 +46,7 @@
     <description>
         core clock stop failed.
     </description>
-    <ffdc>CORE_TARGET</ffdc>
     <ffdc>CORECLKSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
   <hwpError>
@@ -69,30 +54,7 @@
     <description>
         core clock stop timed out.
     </description>
-    <ffdc>CORE_TARGET</ffdc>
     <ffdc>CORECPLTSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_CORE</childType>
-        <childNumber>CORE_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
 </hwpErrors>
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_l2_stopclocks_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_l2_stopclocks_errors.xml
index 774b0bc2..18a5dd06 100644
--- a/src/import/chips/p9/procedures/xml/error_info/p9_hcd_l2_stopclocks_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_hcd_l2_stopclocks_errors.xml
@@ -38,30 +38,7 @@
     <description>
         L2 clock stop failed.
     </description>
-    <ffdc>EQ_TARGET</ffdc>
     <ffdc>EQCLKSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
   <hwpError>
@@ -69,30 +46,7 @@
     <description>
         L2 clock stop timed out.
     </description>
-    <ffdc>EQ_TARGET</ffdc>
     <ffdc>EQCPLTSTAT</ffdc>
-    <callout>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-      <priority>HIGH</priority>
-    </callout>
-    <deconfigure>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </deconfigure>
-    <gard>
-      <childTargets>
-        <parent>PROC_CHIP_IN_ERROR</parent>
-        <childType>TARGET_TYPE_EQ</childType>
-        <childNumber>EQ_NUMBER_IN_ERROR</childNumber>
-      </childTargets>
-    </gard>
   </hwpError>
   <!-- ********************************************************************* -->
 </hwpErrors>