HWP-CORE/CACHE: Update Istep 4 procedures regressed on model 34

Change-Id: Ia88b64463b0b911aa0882db20b85eda7a30571d6
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/22225
Tested-by: Jenkins Server
Tested-by: PPE CI
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/24656

12 files changed, 468 insertions, 435 deletions

diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache.H b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache.H
index dc2f1563..abd43fad 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache.H
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -24,7 +24,7 @@
 /// *HWP FW Owner    : Sangeetha T S  <sangeet2@in.ibm.com>
 /// *HWP Team        : PM
 /// *HWP Consumed by : SBE:SGPE
-/// *HWP Level       : 1
+/// *HWP Level       : 2
 ///
 
 #ifndef __P9_HCD_CACHE_H__
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
index 0c691c2e..9a2628b8 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,145 +20,93 @@
 /// @file  p9_hcd_cache_arrayinit.C
 /// @brief EX Initialize arrays
 ///
-/// *HWP HWP Owner   : David Du       <daviddu@us.ibm.com>
-/// *HWP FW Owner    : Sangeetha T S  <sangeet2@in.ibm.com>
-/// *HWP Team        : PM
-/// *HWP Consumed by : SBE:SGPE
-/// *HWP Level       : 1
-///
 /// Procedure Summary:
 ///   Use ABIST engine to zero out all arrays
 ///   Upon completion, scan0 flush all rings
 ///     except Vital, Repair, GPTR, TIME and DPLL
 ///
 
+// *HWP HWP Owner          : David Du      <daviddu@us.ibm.com>
+// *HWP Backup HWP Owner   : Greg Still    <stillgs@us.ibm.com>
+// *HWP FW Owner           : Sangeetha T S <sangeet2@in.ibm.com>
+// *HWP Team               : PM
+// *HWP Consumed by        : SBE:SGPE
+// *HWP Level              : 2
+
 //-----------------------------------------------------------------------------
 // Includes
 //-----------------------------------------------------------------------------
-#include <fapi2.H>
-//#include <common_scom_addresses.H>
-//will be replaced with real scom address header file
+#include <p9_perv_sbe_cmn.H>
+#include <p9_hcd_common.H>
 #include "p9_hcd_cache_arrayinit.H"
 
 //-----------------------------------------------------------------------------
 // Constant Definitions
 //-----------------------------------------------------------------------------
+enum P9_HCD_CACHE_ARRAYINIT_Private_Constants
+{
+    LOOP_COUNTER = 0x0000000000042FFF,
+    REGIONS_EXCEPT_VITAL = 0x7FF,
+    REGIONS_FOR_PERV = 0x400,
+    SCAN_TYPES_EXCEPT_TIME_GPTR_REPR = 0xDCF,
+    SELECT_EDRAM = 0x0,
+    SELECT_SRAM = 0x1,
+    START_ABIST_MATCH_VALUE = 0x0000000F00000000
+};
 
 //-----------------------------------------------------------------------------
 // Procedure: Initialize Cache Arrays
 //-----------------------------------------------------------------------------
 
-extern "C"
+fapi2::ReturnCode
+p9_hcd_cache_arrayinit(
+    const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
 {
 
-    fapi2::ReturnCode
-    p9_hcd_cache_arrayinit(
-        const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
-    {
-
-#if 0
-        fapi2::buffer<uint64_t> data;
-        uint32_t                scan;
-        uint32_t                loop;
-
-        // Procedure Prereq :    P0 is pointing to the targeted EX chiplet
-        // submodules:
-        // seeprom_array_init_module
-        // ex_scan0
-
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              *** Array Init and Scan0 Cleanup for EX Chiplets ***");
-
-        // SBE Address Base Register Setups
-        // Setup PRV_BASE_ADDR1; points to selected EX chiplet
-        // - mr      P1, P0
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              Copy selected EX info from P0 to P1");
-
-        // Step 1: Array Init for selected EX chiplet
-        // ARRAY INIT module -> see p9_sbe_tp_array_init.S
-        //
-        // At entry:
-        //
-        //    P1 : The chiplet ID/Multicast Group
-        //    D1 : Clock Regions for Array Init
-        //
-        // At exit:
-        //
-        //    P0, D0, D1, CTR  : destroyed
-        //    P1, A0, A1       : maintained
-        //
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              Calling Array Init Subroutine");
-
-        // >>> IPL/Winkle
-        // \bug Need to exclude DPLL ring for IPL
-        // li      D1, SCAN_ALLREGIONEXVITAL
-        // = li      D1, SCAN_CLK_ALLEXDPLL
-        scan = SCAN_CLK_ALLEXDPLL;
-
-        // Execute the array init
-        // = bsr     seeprom_array_init_module
-        seeprom_array_init_module(scan);
-
-        // Restore P0 with selected EX chiplet info
-        // - mr      P0, P1
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              Copy selected EX info back from P1 to P0");
-
-        // Step 2: Scan0 for selected EX chiplet except PRV, GPTR, TIME and DPLL
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              Calling Scan0 Subroutine");
-
-        // taken from p9_sbe_ex_chiplet_init
-
-        // >>> IPL/Winkle scan flush - all except vital
-
-        // Hook to bypass in Sim
-        // - hooki 0, 0xFF02
-        // - ifslwcntlbitset P9_SLW_SKIP_FLUSH,1f
-
-        FAPI_DBG("EX Init: Scan0 Module executed: \
-              Scan all except vital, DPL, GPTR, and TIME scan chains");
-
-        // Each scan0 will rotate the ring 8191 latches (2**13 - 1) and the
-        // the longest ring is defined by P9_SCAN0_FUNC_REPEAT.  When the design
-        // ALWAYS has all stumps less than 8191, the repeat (eg .rept) can be
-        // removed.
-        // Implementation note:  this is not done in a loop (or included in the
-        // ex_scan0_module itself) as the D0 and D1 registers are used in
-        // ex_scan0_module and there is no convenient place to temporaily store
-        // the 2-64b values values. Argueably, PIBMEM could be used for this
-        // but was not utilized.
-
-        // \bug remove DPLL ring
-        // ex_scan0 SCAN_ALL_BUT_VITALDPLLGPTRTIME, SCAN_CLK_ALL
-        // = .rept P9_SCAN0_FUNC_REPEAT
-        // = ex_scan0 SCAN_ALL_BUT_VITALDPLLGPTRTIME, SCAN_CLK_ALLEXDPLL
-        // = .endr
-        for(loop = 0; loop < P9_SCAN0_FUNC_REPEAT; loop++)
-        {
-            ex_scan0(SCAN_ALL_BUT_VITALDPLLGPTRTIME, SCAN_CLK_ALLEXDPLL);
-        }
-
-        // - 1:
-
-        FAPI_INF("<p9_hcd_cache_arrayinit> : \
-              *** End of Procedure ***");
-
-        return fapi2::FAPI2_RC_SUCCESS;
-
-        FAPI_CLEANUP();
-        return fapi2::FAPI2_RC_PLAT_ERR_SEE_DATA;
+    FAPI_INF(">>p9_hcd_cache_arrayinit");
+
+#ifndef P9_HCD_STOP_SKIP_ARRAYINIT
+
+    fapi2::Target<fapi2::TARGET_TYPE_PERV> l_perv =
+        i_target.getParent<fapi2::TARGET_TYPE_PERV>();
+
+    FAPI_DBG("Arrayinit all regions except Vital");
+    FAPI_TRY(p9_perv_sbe_cmn_array_init_module(l_perv,
+             REGIONS_EXCEPT_VITAL,
+             LOOP_COUNTER,
+             SELECT_SRAM,
+             SELECT_EDRAM,
+             START_ABIST_MATCH_VALUE));
+
+#endif
+
+#ifndef P9_HCD_STOP_SKIP_FLUSH
+
+    //--------------------------------------------
+    // perform scan0 module for pervasive chiplet
+    //--------------------------------------------
+    // Each scan0 will rotate the ring 8191 latches (2**13 - 1) and the longest
+    // ring is defined by P9_HCD_SCAN_FUNC_REPEAT. When the design ALWAYS has
+    // all stumps less than 8191, the loop can be removed.
+    FAPI_DBG("Scan0 all except Vital/ANEP/DPLL, GPTR/REPR/TIME scan chains");
+
+    for(uint32_t l_loop = 0; l_loop < P9_HCD_SCAN_FUNC_REPEAT; l_loop++)
+        FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
+                                              p9hcd::SCAN0_REGION_ALL_BUT_ANEP_PLL,
+                                              p9hcd::SCAN0_TYPE_ALL_BUT_GPTR_REPR_TIME));
+
+#endif
 
+#if not defined(P9_HCD_STOP_SKIP_FLUSH) || not defined(P9_HCD_STOP_SKIP_ARRAYINIT)
+fapi_try_exit:
 #endif
 
-        return fapi2::FAPI2_RC_SUCCESS;
+    FAPI_INF("<<p9_hcd_cache_arrayinit");
 
-    } // Procedure
+    return fapi2::current_err;
 
+}
 
-} // extern C
 
 
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.H b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.H
index d61cee2d..4260e68f 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.H
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_arrayinit.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,23 +20,26 @@
 /// @file  p9_hcd_cache_arrayinit.H
 /// @brief EX Initialize arrays
 ///
-/// *HWP HWP Owner   : David Du       <daviddu@us.ibm.com>
-/// *HWP FW Owner    : Sangeetha T S  <sangeet2@in.ibm.com>
-/// *HWP Team        : PM
-/// *HWP Consumed by : SBE:SGPE
-/// *HWP Level       : 1
-///
+
+// *HWP HWP Owner          : David Du      <daviddu@us.ibm.com>
+// *HWP Backup HWP Owner   : Greg Still    <stillgs@us.ibm.com>
+// *HWP FW Owner           : Sangeetha T S <sangeet2@in.ibm.com>
+// *HWP Team               : PM
+// *HWP Consumed by        : SBE:SGPE
+// *HWP Level              : 2
 
 #ifndef __P9_HCD_CACHE_ARRAYINIT_H__
 #define __P9_HCD_CACHE_ARRAYINIT_H__
 
-extern "C"
-{
+#include <fapi2.H>
 
 /// @typedef p9_hcd_cache_arrayinit_FP_t
 /// function pointer typedef definition for HWP call support
-    typedef fapi2::ReturnCode (*p9_hcd_cache_arrayinit_FP_t) (
-        const fapi2::Target<fapi2::TARGET_TYPE_EQ>&);
+typedef fapi2::ReturnCode (*p9_hcd_cache_arrayinit_FP_t) (
+    const fapi2::Target<fapi2::TARGET_TYPE_EQ>&);
+
+extern "C"
+{
 
 /// @brief EX Initialize arrays
 ///
@@ -50,7 +53,6 @@ extern "C"
     p9_hcd_cache_arrayinit(
         const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target);
 
-
-} // extern C
+}
 
 #endif  // __P9_HCD_CACHE_ARRAYINIT_H__
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_init.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_init.C
index 71228bd8..d6de25fb 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_init.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_init.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -62,11 +62,14 @@ p9_hcd_cache_chiplet_init(
     // Each scan0 will rotate the ring 8191 latches (2**13 - 1) and the longest
     // ring is defined by P9_HCD_SCAN_FUNC_REPEAT. When the design ALWAYS has
     // all stumps less than 8191, the loop can be removed.
-    fapi2::Target<fapi2::TARGET_TYPE_PERV> l_target;
+    uint32_t l_loop;
+    fapi2::Target<fapi2::TARGET_TYPE_PERV> l_perv =
+        i_target.getParent<fapi2::TARGET_TYPE_PERV>();
+
     FAPI_DBG("Scan0 all except Vital/DPLL, GPTR/TIME/REPR scan chains");
 
-    for(uint32_t l_loop = 0; l_loop < P9_HCD_SCAN_FUNC_REPEAT; l_loop++)
-        FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_target,
+    for(l_loop = 0; l_loop < P9_HCD_SCAN_FUNC_REPEAT; l_loop++)
+        FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
                                               p9hcd::SCAN0_REGION_ALL_BUT_PLL,
                                               p9hcd::SCAN0_TYPE_ALL_BUT_GPTR_REPR_TIME));
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
index 070b556a..483001be 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_chiplet_reset.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -22,6 +22,11 @@
 ///
 /// Procedure Summary:
 ///   Reset quad chiplet logic
+///   Clocking:
+///    - setup cache sector buffer strength,
+///      pulse mode and pulsed mode enable values
+///      (attribute dependency Nimbus/Cumulus)
+///    - Drop glsmux async reset
 ///   Scan0 flush entire cache chiplet
 ///
 
@@ -62,62 +67,62 @@ p9_hcd_cache_chiplet_reset(
     // If there is an unused, powered-off cache chiplet which needs to be
     // configured in the following steps to setup the PCB endpoint.
 
-    FAPI_DBG("Init NETWORK_CONTROL0, step needed for hotplug");
+    FAPI_DBG("Init NET_CTRL0[0,1,3,4,5,12,13,14,18], step needed for hotplug");
     fapi2::buffer<uint64_t> l_data64 = p9hcd::NET_CTRL0_INIT_VECTOR;
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0, l_data64));
 
-    FAPI_DBG("Init L3 Glitchless Mux Reset/Select via CLOCK_GRID_CTRL[0:3]");
-    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR_OR, MASK_OR(0, 4, 0x8)));
+    FAPI_DBG("Assert Progdly+DCC Bypass,L2 DCC Reset via NET_CTRL1[1,2,23,24]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL1_WOR, MASK_OR(1, 2, 3)));
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL1_WOR, MASK_OR(23, 2, 3)));
 
-    FAPI_DBG("Init L2 Glitchless Mux Reset/Select via ANALOG_CLK_CTRL[16:19]");
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_OR, MASK_OR(16, 4, 0x8)));
+    FAPI_DBG("Assert Skew Adjust Reset via NET_CTRL0[2]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(2)));
 
-    FAPI_DBG("Clear L3 Glitchless Mux Async Reset via CLOCK_GRID_CTRL[0]");
-    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR_CLEAR, MASK_SET(0)));
+    FAPI_DBG("Set DPLL ff_bypass to 1 via QPPM_DPLL_CTRL[2]");
+    FAPI_TRY(putScom(i_target, EQ_QPPM_DPLL_CTRL_OR, MASK_SET(2)));
 
-    FAPI_DBG("Clear L2 Glitchless Mux Async Reset via ANALOG_CLK_CTRL[16]");
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_CLEAR, MASK_SET(16)));
+    FAPI_DBG("Drop Vital Thold via NET_CTRL0[16]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(16)));
 
-    FAPI_DBG("Clear PCB Endpoint Reset via NET_CTRL0[1]");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(1)));
+    FAPI_DBG("Init L3 glsmux reset/select via CLOCK_GRID_CTRL[0:3]");
+    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR, MASK_OR(0, 4, 0)));
 
-    FAPI_DBG("Reset PCB Slave Error Register");
-    FAPI_TRY(putScom(i_target, EQ_ERROR_REG, MASK_ALL));
+    FAPI_DBG("Init L2 glsmux reset/select via EXCLK_GRID_CTRL[32:35]");
+    FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_CLEAR, MASK_CLR(32, 4, 0xF)));
 
-    /// @todo FAPI_DBG("Remove pervasive ECO fence;");
-    /// @todo FAPI_DBG("Remove logical pervasive/pcbs-pm fence");
+    FAPI_DBG("Clear PCB Endpoint Reset via NET_CTRL0[1]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(1)));
 
-    FAPI_DBG("Remove Chiplet Electrical Fence via NET_CTRL0[26]");
+    FAPI_DBG("Remove chiplet electrical fence via NET_CTRL0[26]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(26)));
 
-    FAPI_DBG("Configure HANG_PULSE1 for Chiplet Hang Counters");
-    l_data64 = p9hcd::HANG_PULSE1_INIT_VECTOR;
-    FAPI_TRY(putScom(i_target, EQ_HANG_PULSE_1_REG, l_data64));
+    FAPI_DBG("Remove PCB fence via NET_CTRL0[25]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(25)));
 
 #ifndef P9_HCD_STOP_SKIP_FLUSH
     // Putting in block to avoid c++ crosses initialization compile error
     {
-        /// @todo Drop the core2cache and cache2core fences to allow for L2 scanning
-
         //--------------------------------------------
         // Perform scan0 module for pervasive chiplet
         //--------------------------------------------
-        // Each scan0 will rotate the ring 8191 latches (2**13 - 1) and the longest
-        // ring is defined by P9_HCD_SCAN_FUNC_REPEAT. When the design ALWAYS has
-        // all stumps less than 8191, the repeat can be removed.
+        //Each scan0 will rotate the ring 8191 latches (2**13-1) and the longest
+        //ring is defined by P9_HCD_SCAN_FUNC_REPEAT. When the design ALWAYS has
+        //all stumps less than 8191, the repeat can be removed.
         uint32_t l_loop;
-        fapi2::Target<fapi2::TARGET_TYPE_PERV> l_target;
+        fapi2::Target<fapi2::TARGET_TYPE_PERV> l_perv =
+            i_target.getParent<fapi2::TARGET_TYPE_PERV>();
+
         FAPI_DBG("Scan0 the GPTR/TIME/REPR rings");
 
         for(l_loop = 0; l_loop < P9_HCD_SCAN_GPTR_REPEAT; l_loop++)
-            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_target,
+            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
                                                   p9hcd::SCAN0_REGION_ALL,
                                                   p9hcd::SCAN0_TYPE_GPTR_REPR_TIME));
 
         FAPI_DBG("Scan0 all but the GPTR/TIME/REPR rings");
 
         for(l_loop = 0; l_loop < P9_HCD_SCAN_FUNC_REPEAT; l_loop++)
-            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_target,
+            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
                                                   p9hcd::SCAN0_REGION_ALL,
                                                   p9hcd::SCAN0_TYPE_ALL_BUT_GPTR_REPR_TIME));
     }
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
index 36d7ab08..133da10b 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,85 +20,63 @@
 /// @file  p9_hcd_cache_dpll_setup.C
 /// @brief Quad DPLL Setup
 ///
+/// Procedure Summary:
+///   Note:
+///     Initfiles in procedure defined on VBU ENGD wiki (TODO add link)
+///     DPLL tune bits are not dependent on frequency
+///     Frequency is controlled by the Quad PPM
+///       Actual frequency value for boot is stored into the Quad PPM by
+///         p9_hcd_setup_evid.C in istep 2
+///       In real cache STOP exit, the frequency value is persistent
+///
+///   Pre-Scan:
+///
+///   Scan:
+///     (TODO) Set clock controller scan ratio to 1:1 as this is done at refclk
+///     (TODO) scan0 (region = DPLL and ANEP, scan_type = GPTR)
+///     (TODO) scan0 (region = DPLL and ANEP, scan_type = FUNC)
+///     (TODO) Set clock controller scan ratio to 8:1 for future scans
+///
+///   Setup:
+///     (TODO) set DPLL FREQ CTRL regitster
+///     (TODO) set DPLL CTRL register
+///     (Done) Drop DPLL test mode;
+///     (Done) Drop DPLL into Reset;
+///     (Done) Start DPLL clock via quad clock controller
+///     (Done) Check for DPLL lock, Timeout: 200us
+///     (Done) Remove DPLL bypass
+///     (Done) Switch cache glitchless mux to use the DPLL
+///     (Done) Drop ff_bypass to enable slewing
+///
+/// 1) reset, dpll_func_clksel, and all SL_HLD inputs are asserted
+/// 2) If grid clock connected to dpll clkout,
+///    bypass also has to be asserted to allow refclk on grid
+
 // *HWP HWP Owner          : David Du       <daviddu@us.ibm.com>
 // *HWP Backup HWP Owner   : Greg Still     <stillgs@us.ibm.com>
 // *HWP FW Owner           : Sangeetha T S  <sangeet2@in.ibm.com>
 // *HWP Team               : PM
 // *HWP Consumed by        : SBE:SGPE
 // *HWP Level              : 2
-//
-// Procedure Summary:
-//   Note:
-//     Initfiles in procedure defined on VBU ENGD wiki (TODO add link)
-//     DPLL tune bits are not dependent on frequency
-//     Frequency is controlled by the Quad PPM
-//       Actual frequency value for boot is stored into the Quad PPM by
-//         p9_hcd_setup_evid.C in istep 2
-//       In real cache STOP exit, the frequency value is persistent
-//
-//   Pre-Scan:
-//     (Done) +Drop DPLL test mode;
-//     (Done) +Drop DPLL into Reset;
-//     (Done) Put DPLL into bypass;
-//     (Done) +Set ff_bypass = 1;
-//     (Done) Set DPLL syncmux sel;
-//
-//   Scan:
-//     (TBD)  Set clock controller scan ratio to 1:1
-//            as this is done at refclk speeds
-//     (TBD)  Load the EX DPLL scan ring
-//     (TBD)  Set clock controller scan ratio to 8:1 for future scans
-//     (Done) +Start DPLL clock via quad clock controller
-//
-//   Setup:
-//     (Done) +Ensure ff_bypass = 1, should this be scaned?
-//     (Done) Enable the DPLL in the correct mode: non-dynamic(b1=0 by default?)
-//     (Done) Slew rate established per DPLL team
-//     (Done) +Clear DPLL syncmux sel
-//     (TBD)  *Take the cache glitchless mux out of reset (in chiplet_reset?)
-//     (Done) Remove DPLL bypass
-//     (TBD)  Drop DPLL Tholds (still in p9?)
-//     (Done) Check for DPLL lock, Timeout: 200us
-//     (Done) +Recycle DPLL in and out of bypass
-//     (Done) Switch cache glitchless mux to use the DPLL (in startclocks?)
-//     (Done) +Drop ff_bypass to enable slewing
-//
-//   Notion: + means the step isnt on p9 ipl flow doc but     in p8 code
-//           * means the step is   on p9 ipl flow doc but not in p8 code
 
 //-----------------------------------------------------------------------------
 // Includes
 //-----------------------------------------------------------------------------
+
 #include <p9_quad_scom_addresses.H>
+#include <p9_perv_sbe_cmn.H>
 #include <p9_hcd_common.H>
 #include "p9_hcd_cache_dpll_setup.H"
 
 //-----------------------------------------------------------------------------
 // Constant Definitions
 //-----------------------------------------------------------------------------
-// Setup DPLL like this: (from dpll_spec_14nm_20140107)
-//  (marked as IPL from IPL document)
-// Initial State:  (Do we need these (1-3) in IPL?)
-// 1) reset, dpll_func_clksel, and all SL_HLD inputs are asserted
-// 2) If grid clock connected to dpll clkout,
-//    bypass also has to be asserted to allow refclk on grid
-// 3) Scan init:
-//    - All scan rings in DPLL are scanned with zeroes
-//    - Scan in VPD table values into the DPLL_MODE ring (config registers)
-//      lf_param_dc<0:15> = b1010001010000000#
-//      will change based on HW characterization
-//     lf_sdorder_dc<0:1> = b10
-//     cd_div124_dc<0:1> = b10
-//     cd_dpllout124_dc<0:1> = b10
-//     vc_vtune_dc<0:2> = 0b100 # may change based on HW characterization
-//     ref_div_dc<0:5> = b000001
-//     ref_div_vreg_dc<0:4> = b000010
-//     refclk_sel_dc = b1
-//     All configuration register bits not specified should be set to b0
 
 enum P9_HCD_CACHE_DPLL_SETUP_CONSTANTS
 {
-    CACHE_DPLL_LOCK_TIMEOUT_IN_MS = 1
+    CACHE_DPLL_LOCK_TIMEOUT_IN_MS      = 1,
+    CACHE_DPLL_CLK_START_TIMEOUT_IN_MS = 1,
+    CACHE_ANEP_CLK_START_TIMEOUT_IN_MS = 1
 };
 
 //-----------------------------------------------------------------------------
@@ -115,27 +93,71 @@ p9_hcd_cache_dpll_setup(
     FAPI_INF(">>p9_hcd_cache_dpll_setup");
 
     // --------------
-    // PRE-SCAN
+    // DPLL SCAN
     // --------------
+    FAPI_DBG("Set scan ratio to 1:1 in bypass mode");
+    FAPI_TRY(getScom(i_target, EQ_OPCG_ALIGN, l_data64));
+    l_data64.insertFromRight<47, 5>(0x0);
+    FAPI_TRY(putScom(i_target, EQ_OPCG_ALIGN, l_data64));
 
-    FAPI_DBG("Drop DPLL Test Mode and Reset");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_AND(3, 2, 0)));
+#ifndef P9_HCD_STOP_SKIP_FLUSH
+    // Putting in block to avoid c++ crosses initialization compile error
+    {
+        //--------------------------------------------
+        // Perform scan0 module for pervasive chiplet
+        //--------------------------------------------
+        //Each scan0 will rotate the ring 8191 latches (2**13-1) and the longest
+        //ring is defined by P9_HCD_SCAN_FUNC_REPEAT. When the design ALWAYS has
+        //all stumps less than 8191, the repeat can be removed.
+        uint32_t l_loop;
+        fapi2::Target<fapi2::TARGET_TYPE_PERV> l_perv =
+            i_target.getParent<fapi2::TARGET_TYPE_PERV>();
+
+        FAPI_DBG("Scan0 the DPLL/ANEP rings(GPTR type)");
+
+        for(l_loop = 0; l_loop < P9_HCD_SCAN_GPTR_REPEAT; l_loop++)
+            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
+                                                  p9hcd::SCAN0_REGION_DPLL_ANEP,
+                                                  p9hcd::SCAN0_TYPE_GPTR));
+
+        FAPI_DBG("Scan0 all DPLL/ANEP rings(func type)");
+
+        for(l_loop = 0; l_loop < P9_HCD_SCAN_FUNC_REPEAT; l_loop++)
+            FAPI_TRY(p9_perv_sbe_cmn_scan0_module(l_perv,
+                                                  p9hcd::SCAN0_REGION_DPLL_ANEP,
+                                                  p9hcd::SCAN0_TYPE_FUNC));
+    }
+#endif
+
+#ifndef P9_HCD_STOP_SKIP_SCAN
 
-    FAPI_DBG("Put DPLL into bypass mode");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR,  MASK_SET(5)));
+    /// @todo putRing(DPLL,FUNC) here
 
-    FAPI_DBG("Put DPLL into Mode 1 by asserting ff_bypass");
-    FAPI_TRY(putScom(i_target, EQ_QPPM_DPLL_CTRL_OR, MASK_SET(2)));
+#endif
 
-    /// @todo Is there a dpllclk_muxsel in p9?
-    FAPI_DBG("Set syncclk_muxsel and dpllclk_muxsel");
-    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_OR, MASK_SET(1)));
+    FAPI_DBG("Set scan ratio to 8:1 in bypass mode");
+    FAPI_TRY(getScom(i_target, EQ_OPCG_ALIGN, l_data64));
+    l_data64.insertFromRight<49, 3>(0x7);
+    FAPI_TRY(putScom(i_target, EQ_OPCG_ALIGN, l_data64));
 
     // --------------
-    // DPLL SCAN
+    // DPLL SETUP
     // --------------
-    /// @todo scan dpll here
-    /// @todo start dpll clock here?
+
+    FAPI_DBG("Ensure DPLL in Mode 1, and set slew rate to a modest value");
+    l_data64.flush<0>().setBit<2>().insertFromRight<6, 10>(0x40);
+    FAPI_TRY(putScom(i_target, EQ_QPPM_DPLL_CTRL_OR, l_data64));
+
+    FAPI_DBG("Drop DPLL Test Mode and Reset");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_AND(3, 2, 0)));
+
+    FAPI_DBG("Drop DPLL Clock Region Fence");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, MASK_SET(14)));
+
+    // ----------------
+    // Start DPLL clock
+    // ----------------
+
     FAPI_DBG("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
     FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
 
@@ -150,29 +172,25 @@ p9_hcd_cache_dpll_setup(
     l_data64 = p9hcd::CLK_START_REGION_DPLL_THOLD_ALL;
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
-    // --------------
-    // DPLL SETUP
-    // --------------
-
-    // This is necessary to ensure that the DPLL is in Mode 1.
-    // If not, the lock times will go from ~30us to 3-5ms
-    FAPI_DBG("Ensure DPLL in Mode 1, and set slew rate to a modest value");
-    l_data64.flush<0>().setBit<2>().insertFromRight<6, 10>(0x40);
-    FAPI_TRY(putScom(i_target, EQ_QPPM_DPLL_CTRL_OR, l_data64));
-
-    /// @todo Is there a dpllclk_muxsel in p9?
-    FAPI_DBG("Clear syncclk_muxsel and dpllclk_muxsel");
-    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(1)));
-
-    /// @todo Already done in chiplet_reset?
-    FAPI_DBG("Drop glitchless mux async reset");
+    FAPI_DBG("Poll for DPLL clock running");
+    l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
+                CACHE_DPLL_CLK_START_TIMEOUT_IN_MS;
 
-    FAPI_DBG("Take DPLL out of bypass");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(5)));
+    // Read Clock Status Register (Cache chiplet)
+    // check for bits 4:14 eq. zero, no tholds on
+    do
+    {
+        FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+    }
+    while(((l_data64 & BIT64(14)) != 0) && ((--l_timeout) != 0));
 
-    /// @todo Is there a dpll_thold in p9?
-    FAPI_DBG("Drop internal DPLL THOLD");
+    FAPI_ASSERT((l_timeout != 0),
+                fapi2::PMPROC_DPLLCLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
+                "DPLL Clock Start Timeout");
+    FAPI_DBG("DPLL clock running now");
 
+    // This is necessary to ensure that the DPLL is in Mode 1(ff_bypass = 1)
+    // If not, the lock times will go from ~30us to 3-5ms
     /// @todo Determine whether or not we should POLL instead of put delay here.
     FAPI_DBG("Wait for DPLL to lock");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
@@ -189,20 +207,62 @@ p9_hcd_cache_dpll_setup(
                 fapi2::PMPROC_DPLL_LOCK_TIMEOUT()
                 .set_EQQPPMDPLLSTAT(l_data64),
                 "DPLL lock timeout");
+    FAPI_DBG("DPLL is locked");
 
-    FAPI_DBG("Recycle DPLL in and out of bypass");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR,  MASK_SET(5)));
+    FAPI_DBG("Take DPLL out of bypass");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(5)));
 
-    FAPI_DBG("DPLL is locked");
+    FAPI_DBG("Switch L3 glsmux select to DPLL output");
+    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR, MASK_SET(3)));
 
-    /// @todo Later done in startclocks?
-    FAPI_DBG("Set glitchless mux select to dpll");
-    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR_OR, MASK_SET(3)));
+    FAPI_DBG("Switch L2 glsmux select to DPLL output");
+    FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_OR, MASK_OR(34, 2, 3)));
 
     FAPI_DBG("Drop ff_bypass to switch into slew-controlled mode")
     FAPI_TRY(putScom(i_target, EQ_QPPM_DPLL_CTRL_CLEAR, MASK_SET(2)));
-
+    /*
+        FAPI_DBG("Drop ANEP Clock Region Fence");
+        FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, MASK_SET(10)));
+
+        // ----------------
+        // Start ANEP clock
+        // ----------------
+
+        FAPI_DBG("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
+        FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
+
+        FAPI_DBG("Start clock(arrays+nsl clock region) via CLK_REGION");
+        l_data64 = p9hcd::CLK_START_REGION_ANEP_THOLD_NSL_ARY;
+        FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
+
+        /// @todo parameterize delay
+        FAPI_TRY(fapi2::delay(0, 1000000));
+
+        FAPI_DBG("Start clock(sl+refresh clock region) via CLK_REGION");
+        l_data64 = p9hcd::CLK_START_REGION_ANEP_THOLD_ALL;
+        FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
+
+        FAPI_DBG("Poll for ANEP clock running");
+        l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
+                    CACHE_ANEP_CLK_START_TIMEOUT_IN_MS;
+
+        // Read Clock Status Register (Cache chiplet)
+        // check for bits 4:14 eq. zero, no tholds on
+        do
+        {
+            FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
+        }
+        while(((l_data64 & BIT64(10)) != 0) && ((--l_timeout) != 0));
+
+        FAPI_ASSERT((l_timeout != 0),
+                    fapi2::PMPROC_ANEPCLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
+                    "ANEP Clock Start Timeout");
+        FAPI_DBG("ANEP clock running now");
+
+        // @todo is this bit really skew adjust? note it is set in chiplet reset
+        FAPI_DBG("Release skew adjust reset");
+        FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(2)));
+    */
 fapi_try_exit:
 
     FAPI_INF("<<p9_hcd_cache_dpll_setup");
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.H b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.H
index 0e3ceb09..ec7f682c 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.H
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_dpll_setup.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -25,7 +25,7 @@
 /// *HWP FW Owner           : Sangeetha T S  <sangeet2@in.ibm.com>
 /// *HWP Team               : PM
 /// *HWP Consumed by        : SBE:SGPE
-/// *HWP Level              : 1
+/// *HWP Level              : 2
 ///
 
 #ifndef __P9_HCD_CACHE_DPLL_SETUP_H__
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.C
index b1651256..8e0eca88 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,25 +20,29 @@
 /// @file  p9_hcd_cache_poweron.C
 /// @brief Cache Chiplet Power-on
 ///
+/// Procedure Summary:
+///   Set glsmux async reset
+///   Set DPLL ff_bypass
+///   Command the cache PFET controller to power-on
+///   Check for valid power on completion
+///   Polled Timeout:  100us
+
 // *HWP HWP Owner          : David Du       <daviddu@us.ibm.com>
 // *HWP Backup HWP Owner   : Greg Still     <stillgs@us.ibm.com>
 // *HWP FW Owner           : Sangeetha T S  <sangeet2@in.ibm.com>
 // *HWP Team               : PM
 // *HWP Consumed by        : SBE:SGPE
 // *HWP Level              : 2
-//
-// Procedure Summary:
-//   Command the cache PFET controller to power-on
-//   Check for valid power on completion
-//   Polled Timeout:  100us
-//
 
 //------------------------------------------------------------------------------
 // Includes
 //------------------------------------------------------------------------------
-#include <fapi2.H>
+
+#include <p9_quad_scom_addresses.H>
+#include <p9_hcd_common.H>
+#include <p9_common_poweronoff.H>
+#include <p9_common_poweronoff.C>
 #include "p9_hcd_cache_poweron.H"
-#include "p9_common_poweronoff.H"
 
 //------------------------------------------------------------------------------
 // Constant Definitions
@@ -52,9 +56,39 @@ fapi2::ReturnCode
 p9_hcd_cache_poweron(
     const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
 {
-    FAPI_TRY(p9_common_poweronoff(i_target, p9power::POWER_ON));
+    FAPI_INF(">>p9_hcd_cache_poweron");
+
+    //--------------------------
+    // Prepare to cache power on
+    //--------------------------
+
+    fapi2::buffer<uint64_t> l_data64;
+
+    FAPI_DBG("Assert PCB fence via NET_CTRL0[25]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(25)));
+
+    FAPI_DBG("Assert chiplet electrical fence via NET_CTRL0[26]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(26)));
+
+    FAPI_DBG("Assert Vital Thold via NET_CTRL0[16]");
+    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_SET(16)));
+
+    FAPI_DBG("Set L2 glsmux reset via EXCLK_GRID_CTRL[32:33]");
+    FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_OR, MASK_OR(32, 2, 0x3)));
+
+    FAPI_DBG("Set L3 glsmux reset via CLOCK_GRID_CTRL[0]");
+    FAPI_TRY(putScom(i_target, EQ_PPM_CGCR, MASK_SET(0)));
+
+    //-----------------------
+    // Power on cache chiplet
+    //-----------------------
+
+    FAPI_DBG("Power on cache chiplet");
+    FAPI_TRY(p9_common_poweronoff<fapi2::TARGET_TYPE_EQ>(i_target, p9power::POWER_ON));
 
 fapi_try_exit:
-    return fapi2::current_err;
 
-} // Procedure
+    FAPI_INF("<<p9_hcd_cache_poweron");
+
+    return fapi2::current_err;
+}
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.H b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.H
index 85f2145e..b7091afe 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.H
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_poweron.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,18 +20,18 @@
 /// @file  p9_hcd_cache_poweron.H
 /// @brief Cache Chiplet Power-on
 
-//
 // *HWP HWP Owner          : David Du       <daviddu@us.ibm.com>
 // *HWP Backup HWP Owner   : Greg Still     <stillgs@us.ibm.com>
 // *HWP FW Owner           : Sangeetha T S  <sangeet2@in.ibm.com>
 // *HWP Team               : PM
 // *HWP Level              : 2
 // *HWP Consumed by        : SBE:SGPE
-//
 
 #ifndef __P9_HCD_CACHE_POWERON_H__
 #define __P9_HCD_CACHE_POWERON_H__
 
+#include <fapi2.H>
+
 /// @typedef p9_hcd_cache_poweron_FP_t
 /// function pointer typedef definition for HWP call support
 typedef fapi2::ReturnCode (*p9_hcd_cache_poweron_FP_t) (
@@ -52,4 +52,5 @@ extern "C"
     p9_hcd_cache_poweron(
         const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target);
 }
+
 #endif  // __P9_HCD_CACHE_POWERON_H__
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
index 90f338fd..79cb5720 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,12 +20,6 @@
 /// @file  p9_hcd_cache_scominit.C
 /// @brief  Cache Customization SCOMs
 ///
-/// *HWP HWP Owner   : David Du       <daviddu@us.ibm.com>
-/// *HWP FW Owner    : Sangeetha T S  <sangeet2@in.ibm.com>
-/// *HWP Team        : PM
-/// *HWP Consumed by : SBE:SGPE
-/// *HWP Level       : 1
-///
 /// Procedure Summary:
 ///   Apply any SCOM initialization to the cache
 ///   Stop L3 configuration mode
@@ -33,12 +27,16 @@
 ///   DTS Initialization sequense
 ///
 
+// *HWP HWP Owner          : David Du      <daviddu@us.ibm.com>
+// *HWP Backup HWP Owner   : Greg Still    <stillgs@us.ibm.com>
+// *HWP FW Owner           : Sangeetha T S <sangeet2@in.ibm.com>
+// *HWP Team               : PM
+// *HWP Consumed by        : SBE:SGPE
+// *HWP Level              : 2
+
 //------------------------------------------------------------------------------
 // Includes
 //------------------------------------------------------------------------------
-#include <fapi2.H>
-//#include <common_scom_addresses.H>
-//will be replaced with real scom address header file
 #include "p9_hcd_cache_scominit.H"
 
 //------------------------------------------------------------------------------
@@ -49,76 +47,20 @@
 // Procedure: Cache Customization SCOMs
 //------------------------------------------------------------------------------
 
-extern "C"
-{
-
-    fapi2::ReturnCode
-    p9_hcd_cache_scominit(
-        const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
-    {
-
-#if 0
-        fapi2::buffer<uint64_t> data;
-
-        ///////
-        // NCU
-        ///////
-
-        ///////
-        // L3
-        ///////
-
-        FAPI_DBG("Configuring L3 disable");
-        // - l3_setup L3_SETUP_ACTION_DISABLE, L3_SETUP_UNIT_L3
-
-        ///////
-        // OHA
-        ///////
 
-        FAPI_DBG("Enable OHA to accept idle operations \
-              by removing idle state override");
-        // - setp1_mcreadand D1
-        // = ld      D0, EX_OHA_MODE_REG_RWx1002000D, P1
-        FAPI_TRY(getScom(i_target, EX_OHA_MODE_REG_RWx1002000D, data));
-
-        //FAPI_DBG("Read OHA_MODE value:    0x%16llx", io_pore.d0.read());
-        // = andi    D0, D0, ~BIT(6)
-        data.clearBit<6>();
-
-        //FAPI_DBG("Updated OHA_MODE value: 0x%16llx", io_pore.d0.read());
-        // = std     D0, EX_OHA_MODE_REG_RWx1002000D, P0
-        FAPI_TRY(putScom(i_target, EX_OHA_MODE_REG_RWx1002000D, data));
-
-        // set trace stop on checkstop
-        // Get the ECID to apply trace setup to only Murano DD2+ / Venice
-        // - lpcs    P1, STBY_CHIPLET_0x00000000
-        // - ldandi  D0, PCBMS_DEVICE_ID_0x000F000F, P1, (CFAM_CHIP_ID_CHIP_MASK | CFAM_CHIP_ID_MAJOR_EC_MASK)
-        // - cmpibraeq   D0, 1f, (CFAM_CHIP_ID_MURANO | CFAM_CHIP_ID_MAJOR_EC_1 )
-
-        FAPI_DBG("Configuring EX chiplet trace arrays \
-              to stop on checkstop/recoverable errors")
-        // = sti     GENERIC_DBG_MODE_REG_0x000107C0, P0, BIT(7) | BIT(8)
-        FAPI_TRY(putScom(i_target, GENERIC_DBG_MODE_REG_0x000107C0,
-                         fapi2: buffer<uint64_t>().insertFromRight<7, 2>(0x3)));
-
-        // = sti     GENERIC_DBG_TRACE_REG2_0x000107CB, P0, BIT(17)
-        FAPI_TRY(putScom(i_target, GENERIC_DBG_TRACE_REG2_0x000107CB,
-                         fapi2: buffer<uint64_t>().setBit<17>()));
-        // - 1:
-
-        return fapi2::FAPI2_RC_SUCCESS;
-
-        FAPI_CLEANUP();
-        return fapi2::FAPI2_RC_PLAT_ERR_SEE_DATA;
-
-#endif
+fapi2::ReturnCode
+p9_hcd_cache_scominit(
+    const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target)
+{
+    FAPI_INF(">>p9_hcd_cache_scominit");
 
-        return fapi2::FAPI2_RC_SUCCESS;
+    /// @todo actual scom init content will be required for L3
 
-    } // Procedure
+    FAPI_INF("<<p9_hcd_cache_scominit");
 
+    return fapi2::FAPI2_RC_SUCCESS;
+}
 
-} // extern C
 
 
 
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.H b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.H
index bfb0030c..23c54dc1 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.H
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_scominit.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,24 +20,26 @@
 /// @file  p9_hcd_cache_scominit.H
 /// @brief  Cache Customization SCOMs
 ///
-/// *HWP HWP Owner   : David Du       <daviddu@us.ibm.com>
-/// *HWP FW Owner    : Sangeetha T S  <sangeet2@in.ibm.com>
-/// *HWP Team        : PM
-/// *HWP Consumed by : SBE:SGPE
-/// *HWP Level       : 1
-///
+
+// *HWP HWP Owner          : David Du      <daviddu@us.ibm.com>
+// *HWP Backup HWP Owner   : Greg Still    <stillgs@us.ibm.com>
+// *HWP FW Owner           : Sangeetha T S <sangeet2@in.ibm.com>
+// *HWP Team               : PM
+// *HWP Consumed by        : SBE:SGPE
+// *HWP Level              : 2
 
 #ifndef __P9_HCD_CACHE_SCOMINIT_H__
 #define __P9_HCD_CACHE_SCOMINIT_H__
 
-extern "C"
-{
+#include <fapi2.H>
 
 /// @typedef p9_hcd_cache_scominit_FP_t
 /// function pointer typedef definition for HWP call support
-    typedef fapi2::ReturnCode (*p9_hcd_cache_scominit_FP_t) (
-        const fapi2::Target<fapi2::TARGET_TYPE_EQ>&);
+typedef fapi2::ReturnCode (*p9_hcd_cache_scominit_FP_t) (
+    const fapi2::Target<fapi2::TARGET_TYPE_EQ>&);
 
+extern "C"
+{
 
 /// @brief  Cache Customization SCOMs
 ///
@@ -51,7 +53,6 @@ extern "C"
     p9_hcd_cache_scominit(
         const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target);
 
-
-} // extern C
+}
 
 #endif  // __P9_HCD_CACHE_SCOMINIT_H__
diff --git a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
index 51de3791..373bc31d 100644
--- a/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
+++ b/import/chips/p9/procedures/hwp/cache/p9_hcd_cache_startclocks.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -21,22 +21,28 @@
 /// @brief Quad Clock Start
 ///
 /// Procedure Summary:
-///   Set (to be sure they are set under all conditions) core logical fences
-///   (new for P9)
-///   Drop pervasive thold
 ///   (Done) Setup L3 EDRAM/LCO
-///   Drop pervasive fence
+///   (Done) Setup OPCG_ALIGN
+///   (Done) Drop partial good regional fences(always drop vital and pervasive)
+///   (Done) Drop Vital fence
 ///   (Done) Reset abst clock muxsel, sync muxsel
-///   (Done) Set fabric node/chip ID from the nest version
-///   (Done) Clear clock controller scan register before start
-///   (Done) Start arrays + nsl regions
-///   (Done) Start sl + refresh clock regions
-///   (Done) Check for clocks started
-///   (Done) If not, error
-///   (Done) Clear force align
-///   (Done) Clear flush mode
-///   (Done) Drop the chiplet fence to allow PowerBus traffic
-///
+///   (TODO) Set fabric node/chip ID from the nest version
+///   (Done) module align_chiplets
+///   (Done)  - set flushmode_inh to exit flush mode
+///   (Done)  - set force align
+///   (Done)  - set chiplet_is_aligned
+///   (Done)  - clear chiplet_is_aligned
+///   (Done)  - wait
+///   (Done)  - check chiplet is aligned
+///   (Done)  - clear force align
+///   (Done) module start_clocks
+///   (Done)  - Clear clock controller scan register before start
+///   (Done)  - Start arrays + nsl regions
+///   (Done)  - Start sl + refresh clock regions
+///   (Done) Check for clocks started. If not, error
+///   (Done) Drop the cache to PowerBus logical fence
+///   (Done) Check for cache xstop, If so, error
+///   (Done) Clear flushmode_inh to go into flush mode
 
 // *HWP HWP Owner          : David Du       <daviddu@us.ibm.com>
 // *HWP Backup HWP Owner   : Greg Still     <stillgs@us.ibm.com>
@@ -80,37 +86,81 @@ p9_hcd_cache_startclocks(
     // Prepare to cache startclocks
     // -------------------------------
 
-    /// @todo Drop the Pervasive THOLD, was in p8 code, where in p9?
-
     FAPI_DBG("Enable L3 EDRAM/LCO setup on both EXs");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WOR, MASK_OR(23, 2, 0x3)));
     // 0x0 -> 0x8 -> 0xC -> 0xE -> 0xF to turn on edram
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_OR(0, 8, 0x88)));
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_OR(0, 8, 0xCC)));
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_OR(0, 8, 0xEE)));
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_OR(0, 8, 0xFF)));
-
-    /// @todo get next step from perv, but not in p8 code, necessary?
-    FAPI_DBG("Drop Vital Fence via CPLT_CTRL1[3]");
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(0)));
+    FAPI_TRY(fapi2::delay(12000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(1)));
+    FAPI_TRY(fapi2::delay(1000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(2)));
+    FAPI_TRY(fapi2::delay(4000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(3)));
+    FAPI_TRY(fapi2::delay(1000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(4)));
+    FAPI_TRY(fapi2::delay(12000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(5)));
+    FAPI_TRY(fapi2::delay(1000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(6)));
+    FAPI_TRY(fapi2::delay(4000, 200));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_QCCR_WOR, MASK_SET(7)));
+    FAPI_TRY(fapi2::delay(1000, 200));
+
+    FAPI_DBG("Setup OPCG_ALIGN Register");
+    FAPI_TRY(getScom(i_target, EQ_OPCG_ALIGN, l_data64));
+    l_data64.insertFromRight<0, 4>(0x5).
+    insertFromRight<12, 8>(0x0).
+    insertFromRight<52, 12>(0x10);
+    FAPI_TRY(putScom(i_target, EQ_OPCG_ALIGN, l_data64));
+
+    /// @todo partial good information via attribute, drop all fences for now
+    FAPI_DBG("Drop partial good fence via CPLT_CTRL1");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, 0xFFFF700000000000));
+
+    FAPI_DBG("Drop vital fence via CPLT_CTRL1[3]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, MASK_SET(3)));
 
     FAPI_DBG("Reset abstclk & syncclk muxsel(io_clk_sel) via CPLT_CTRL0[0:1]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_CLR(0, 2, 3)));
 
-    FAPI_DBG("Set abist_mode_dc for cache chiplet(cache recovery) via BIST[1]");
-    FAPI_TRY(getScom(i_target, EQ_BIST, l_data64));
-    FAPI_TRY(putScom(i_target, EQ_BIST, DATA_SET(1)));
-
-    /// @todo set fabric node/chip ID values(read from nest chiplet) still need?
-    /// @todo force chiplet out of flush?
+    /// @todo set fabric node/chip ID values(read from nest chiplet)
+    // FAPI_DBG("setup fabric group/unit/sys ids");
+    // FAPI_TRY(putScom(i_target, EQ_CPLT_CONF0, <attributes>))
 
     // -------------------------------
-    // Start L3 Clock
+    // Align Chiplets
     // -------------------------------
 
-    // @todo put this into dpll_setup?
-    //FAPI_DBG("Switch L3 glsmux to DPLL output");
-    //FAPI_TRY(putScom(i_target, EQ_PPM_CGCR_OR, MASK_SET(3)));
+    FAPI_DBG("Set flushmode_inhibit via CPLT_CTRL0[2]");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_OR, MASK_SET(2)));
+
+    FAPI_DBG("Set force_align via CPLT_CTRL0[3]");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_OR, MASK_SET(3)));
+
+    /// @todo wait for how long
+    /*
+        FAPI_DBG("Poll for cache chiplet aligned");
+        l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
+                    CACHE_CLK_START_TIMEOUT_IN_MS;
+
+        do
+        {
+            FAPI_TRY(getScom(i_target, EQ_CPLT_STAT0, l_data64));
+        }
+        while(((l_data64 & BIT64(9)) != 0) && ((--l_timeout) != 0));
+
+        FAPI_ASSERT((l_timeout != 0),
+                    fapi2::PMPROC_CACHECPLTALIGN_TIMEOUT()
+                    .set_EQCPLTSTAT0(l_data64),
+                    "Cache Chiplets Aligned Timeout");
+        FAPI_DBG("Cache chiplets aligned now");
+    */
+    FAPI_DBG("Clear force_align via CPLT_CTRL0[3]");
+    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(3)));
+
+    // -------------------------------
+    // Start Cache Clock
+    // -------------------------------
 
     FAPI_DBG("Set all bits to zero prior clock start via SCAN_REGION_TYPE");
     FAPI_TRY(putScom(i_target, EQ_SCAN_REGION_TYPE, MASK_ZERO));
@@ -126,49 +176,42 @@ p9_hcd_cache_startclocks(
     l_data64 = p9hcd::CLK_START_REGION_ALL_BUT_DPLL_L2_THOLD_ALL;
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
-    // Read Clock Status Register (Cache chiplet)
-    // check for bits 4:14 eq. zero, no tholds on
     FAPI_DBG("Poll for L3 clock running");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_CLK_START_TIMEOUT_IN_MS;
 
+    // Read Clock Status Register (Cache chiplet)
+    // check for bits 4:14 eq. zero, no tholds on
     do
     {
         FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
     }
-    while(((l_data64 & BITS64(4, 11)) != 0) && ((--l_timeout) != 0));
+    while(((l_data64 & (BITS64(4, 4) | BITS64(10, 4))) != 0) && ((--l_timeout) != 0));
 
     FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CACHECLKSTART_TIMEOUT()
-                .set_EQCLKSTATREGIONS(l_data64),
-                "L3 Clock Start Timeout");
-    FAPI_DBG("L3 clock running now");
+                fapi2::PMPROC_CACHECLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
+                "Cache Clock Start Timeout");
+    FAPI_DBG("Cache clock running now");
 
     // -------------------------------
     // Start L2 Clock
     // -------------------------------
 
-    FAPI_DBG("Switch L2 glsmux to DPLL output");
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_SCOM2, MASK_SET(19)));
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_SCOM2, MASK_SET(39)));
-
     FAPI_DBG("Raise L2 clock sync enable");
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_SCOM2, MASK_SET(13)));
-    FAPI_TRY(putScom(i_target, EQ_QPPM_QACCR_SCOM2, MASK_SET(33)));
+    FAPI_TRY(putScom(i_target, EQ_QPPM_EXCGCR_OR, MASK_OR(36, 2, 3)));
 
     FAPI_DBG("Poll for clock sync done to raise on EX L2s");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_CLK_SYNC_TIMEOUT_IN_MS;
-
+    /*
     do
     {
         FAPI_TRY(getScom(i_target, EQ_QPPM_QACSR, l_data64));
     }
     while(((l_data64 & 0x3) != 3) && ((--l_timeout) != 0));
-
+    */
     FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CACHECLKSYNC_TIMEOUT()
-                .set_EXL2CLKSYNCDONE(l_data64),
+                fapi2::PMPROC_CACHECLKSYNC_TIMEOUT().set_EQPPMQACSR(l_data64),
                 "L2 Clock Sync Timeout");
     FAPI_DBG("EX L2s clock sync done");
 
@@ -183,21 +226,20 @@ p9_hcd_cache_startclocks(
     l_data64 = p9hcd::CLK_START_REGION_L2_THOLD_ALL;
     FAPI_TRY(putScom(i_target, EQ_CLK_REGION, l_data64));
 
-    // Read Clock Status Register (Cache chiplet)
-    // check for bits 4:14 eq. zero, no tholds on
     FAPI_DBG("Poll for L2 clock running");
     l_timeout = (p9hcd::CYCLES_PER_MS / p9hcd::INSTS_PER_POLL_LOOP) *
                 CACHE_CLK_START_TIMEOUT_IN_MS;
 
+    // Read Clock Status Register (Cache chiplet)
+    // check for bits 4:14 eq. zero, no tholds on
     do
     {
         FAPI_TRY(getScom(i_target, EQ_CLOCK_STAT_SL, l_data64));
     }
-    while(((l_data64 & BITS64(4, 11)) != 0) && ((--l_timeout) != 0));
+    while(((l_data64 & BITS64(8, 2)) != 0) && ((--l_timeout) != 0));
 
     FAPI_ASSERT((l_timeout != 0),
-                fapi2::PMPROC_CACHECLKSTART_TIMEOUT()
-                .set_EQCLKSTATREGIONS(l_data64),
+                fapi2::PMPROC_CACHECLKSTART_TIMEOUT().set_EQCLKSTAT(l_data64),
                 "L2 Clock Start Timeout");
     FAPI_DBG("L2 clock running now");
 
@@ -205,28 +247,23 @@ p9_hcd_cache_startclocks(
     // Cleaning up
     // -------------------------------
 
-    /// @todo Check the Global Checkstop FIR of dedicated EX chiplet
-    /// @todo Ben's workaround at model e9025, move clear align/flush to end
-
-    /// @todo what are the remaining fences to drop?
-    FAPI_DBG("Drop remaining fences via CPLT_CTRL1");
-    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL1_CLEAR, 0xEFFF700000000000));
-
     FAPI_DBG("Drop chiplet fence via NET_CTRL0[18]");
     FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(18)));
 
-    FAPI_DBG("Drop fence to allow PCB operations to chiplet via NET_CTRL0[25]");
-    FAPI_TRY(putScom(i_target, EQ_NET_CTRL0_WAND, MASK_UNSET(25)));
-
-    FAPI_DBG("Clear force_align via CPLT_CTRL0[3]");
-    FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(3)));
-
+    /*
+    FAPI_DBG("Check the Global Checkstop FIR");
+    FAPI_TRY(getScom(i_target, EQ_XFIR, l_data64));
+    FAPI_ASSERT(((l_data64 & BITS64(0, 27)) != 0),
+                fapi2::PMPROC_CACHE_XSTOP().set_EQXFIR(l_data64),
+                "Cache Chiplet Checkstop");
+    */
     FAPI_DBG("Clear flushmode_inhibit via CPLT_CTRL0[2]");
     FAPI_TRY(putScom(i_target, EQ_CPLT_CTRL0_CLEAR, MASK_SET(2)));
 
+fapi_try_exit:
+
     FAPI_INF("<<p9_hcd_cache_startclocks");
 
-fapi_try_exit:
     return fapi2::current_err;
 }