Fix bug in cache query state procedure

Change-Id: Ic4869b2d73e90bd213c229fc83b189cb10ad57b6
CQ:SW407497
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/49476
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: Brian T. Vanderpool <vanderp@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/49585
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

4 files changed, 148 insertions, 82 deletions

diff --git a/src/import/chips/p9/procedures/hwp/pm/p9_pm_cme_firinit.C b/src/import/chips/p9/procedures/hwp/pm/p9_pm_cme_firinit.C
index d6b7f3b2b..97835fe80 100644
--- a/src/import/chips/p9/procedures/hwp/pm/p9_pm_cme_firinit.C
+++ b/src/import/chips/p9/procedures/hwp/pm/p9_pm_cme_firinit.C
@@ -240,10 +240,23 @@ fapi2::ReturnCode pm_cme_fir_reset(
         //query_cache_state to safely determine if we can scom
         //the ex targets
         fapi2::ReturnCode l_rc;
-        bool l_l2_is_scanable = false;
-        bool l_l3_is_scanable = false;
-        bool l_l2_is_scomable = false;
-        bool l_l3_is_scomable = false;
+        bool l_l2_is_scanable[MAX_L2_PER_QUAD];
+        bool l_l2_is_scomable[MAX_L2_PER_QUAD];
+        bool l_l3_is_scanable[MAX_L3_PER_QUAD];
+        bool l_l3_is_scomable[MAX_L3_PER_QUAD];
+
+        for (auto cnt = 0; cnt < MAX_L2_PER_QUAD; ++cnt)
+        {
+            l_l2_is_scomable[cnt] = false;
+            l_l2_is_scanable[cnt] = false;
+        }
+
+        for (auto cnt = 0; cnt < MAX_L3_PER_QUAD; ++cnt)
+        {
+            l_l3_is_scanable[cnt] = false;
+            l_l3_is_scomable[cnt] = false;
+        }
+
         uint8_t l_chip_unit_pos;
 
         FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS,
@@ -256,17 +269,25 @@ fapi2::ReturnCode pm_cme_fir_reset(
         FAPI_TRY(l_rc, "ERROR: failed to query cache access state for EQ %d",
                  l_chip_unit_pos);
 
-        //If this cache isnt scommable continue to the next EQ
-        if(!l_l3_is_scomable)
-        {
-            continue;
-        }
 
         auto l_exChiplets = l_eq_chplt.getChildren<fapi2::TARGET_TYPE_EX>
                             (fapi2::TARGET_STATE_FUNCTIONAL);
 
         for(auto l_ex_chplt : l_exChiplets)
         {
+            FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS,
+                                   l_ex_chplt, l_chip_unit_pos),
+                     "ERROR: Failed to get the chip unit pos attribute from the ex");
+
+            //look ex is scommable
+            l_chip_unit_pos = l_chip_unit_pos % 2;
+
+            if ((!(l_l2_is_scomable[l_chip_unit_pos]) &&
+                 !(l_l3_is_scomable[l_chip_unit_pos])))
+            {
+                continue;
+            }
+
             p9pmFIR::PMFir <p9pmFIR::FIRTYPE_CME_LFIR> l_cmeFir(l_ex_chplt);
 
             if (l_firinit_done_flag == 1)
diff --git a/src/import/chips/p9/procedures/hwp/pm/p9_pm_corequad_init.C b/src/import/chips/p9/procedures/hwp/pm/p9_pm_corequad_init.C
index d3a28e270..9d5110bfd 100644
--- a/src/import/chips/p9/procedures/hwp/pm/p9_pm_corequad_init.C
+++ b/src/import/chips/p9/procedures/hwp/pm/p9_pm_corequad_init.C
@@ -371,10 +371,22 @@ fapi2::ReturnCode pm_corequad_reset(
     uint64_t l_address = 0;
     uint32_t l_errMask = 0;
 
-    bool l_l2_is_scanable = false;
-    bool l_l3_is_scanable = false;
-    bool l_l2_is_scomable = false;
-    bool l_l3_is_scomable = false;
+    bool l_l2_is_scanable[MAX_L2_PER_QUAD];
+    bool l_l2_is_scomable[MAX_L2_PER_QUAD];
+    bool l_l3_is_scanable[MAX_L3_PER_QUAD];
+    bool l_l3_is_scomable[MAX_L3_PER_QUAD];
+
+    for (auto cnt = 0; cnt < MAX_L2_PER_QUAD; ++cnt)
+    {
+        l_l2_is_scomable[cnt] = false;
+        l_l2_is_scanable[cnt] = false;
+    }
+
+    for (auto cnt = 0; cnt < MAX_L3_PER_QUAD; ++cnt)
+    {
+        l_l3_is_scanable[cnt] = false;
+        l_l3_is_scomable[cnt] = false;
+    }
 
     auto l_eqChiplets = i_target.getChildren<fapi2::TARGET_TYPE_EQ>
                         (fapi2::TARGET_STATE_FUNCTIONAL);
@@ -446,12 +458,6 @@ fapi2::ReturnCode pm_corequad_reset(
         FAPI_TRY(l_rc, "ERROR: failed to query cache access state for EQ %d",
                  l_chpltNumber);
 
-        if(!l_l3_is_scomable)
-        {
-            //Skip all of the scoms for this EQ if its not scommable
-            continue;
-        }
-
         auto l_exChiplets = l_quad_chplt.getChildren<fapi2::TARGET_TYPE_EX>
                             (fapi2::TARGET_STATE_FUNCTIONAL);
 
@@ -466,6 +472,12 @@ fapi2::ReturnCode pm_corequad_reset(
                      l_ex_chplt);
             FAPI_DBG("EX number = %d", l_chpltNumber);
 
+            if ((!(l_l2_is_scomable[l_chpltNumber % 2]) &&
+                 !(l_l3_is_scomable[l_chpltNumber % 2])))
+            {
+                continue;
+            }
+
             // @todo RTC 179967 PM FFDC HWP update
             // If CME is already halted, create error log and and mark the FFDC
             // dump (below) as such.
diff --git a/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.C b/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.C
index b452cf5f8..828df9d32 100644
--- a/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.C
+++ b/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.C
@@ -70,17 +70,14 @@ const uint32_t SSH_REG_STOP_GATED = 0;
 fapi2::ReturnCode
 p9_query_cache_access_state(
     const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target,
-    bool& o_l2_is_scomable,
-    bool& o_l2_is_scannable,
-    bool& o_l3_is_scomable,
-    bool& o_l3_is_scannable)
+    bool o_l2_is_scomable[MAX_L2_PER_QUAD],
+    bool o_l2_is_scannable[MAX_L2_PER_QUAD],
+    bool o_l3_is_scomable[MAX_L3_PER_QUAD],
+    bool o_l3_is_scannable[MAX_L3_PER_QUAD])
 {
     fapi2::buffer<uint64_t> l_qsshsrc;
     uint32_t l_quadStopLevel = 0;
     fapi2::buffer<uint64_t> l_data64;
-    bool l_is_scomable = 1;
-    uint8_t l_chpltNumber = 0;
-    uint32_t l_exPos = 0;
     uint8_t l_execution_platform = 0;
     uint32_t l_stop_state_reg = 0;
     const fapi2::Target<fapi2::TARGET_TYPE_SYSTEM> FAPI_SYSTEM;
@@ -92,7 +89,7 @@ p9_query_cache_access_state(
     FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_EXECUTION_PLATFORM, FAPI_SYSTEM, l_execution_platform),
              "Error: Failed to get platform");
 
-    if (l_execution_platform == 0x02)
+    if (l_execution_platform == fapi2::ENUM_ATTR_EXECUTION_PLATFORM_FSP)
     {
         l_stop_state_reg =  EQ_PPM_SSHFSP;
     }
@@ -114,71 +111,88 @@ p9_query_cache_access_state(
 
     FAPI_DBG("EQ Stop State: EQ(%d)", l_quadStopLevel);
 
-    //Set all attributes to 1, then clear them based on the stop state
-    o_l2_is_scomable = 1;
-    o_l2_is_scannable = 1;
-    o_l3_is_scomable = 1;
-    o_l3_is_scannable = 1;
+    //Set all attributes to 1, then clear them based on the clock state
+    for (auto cnt = 0; cnt < MAX_L2_PER_QUAD; ++cnt)
+    {
+        o_l2_is_scomable[cnt] = 1;
+        o_l2_is_scannable[cnt] = 1;
+    }
+
+    for (auto cnt = 0; cnt < MAX_L3_PER_QUAD; ++cnt)
+    {
+        o_l3_is_scomable[cnt] = 1;
+        o_l3_is_scannable[cnt] = 1;
+    }
 
     //Looking at the stop states is only valid if quad is stop gated -- else it is fully running
     if (l_qsshsrc.getBit(SSH_REG_STOP_GATED))
     {
-        // STOP8 - Half Quad Deep Sleep
-        //   VSU, ISU are powered off
-        //   IFU, LSU are powered off
-        //   PC, Core EPS are powered off
-        //   L20-EX0 is clocked off if both cores are >= 8
-        //   L20-EX1 is clocked off if both cores are >= 8
-        if (l_quadStopLevel >= 8)
-        {
-            o_l2_is_scomable = 0;
-        }
-
-        // STOP9 - Fast Winkle (lab use only)
-        // Both cores and cache are clocked off
-        if (l_quadStopLevel >= 9)
-        {
-            o_l3_is_scomable = 0;
-        }
 
-        // STOP11 - Deep Winkle
+        // STOP11
         // Both cores and cache are powered off
         if (l_quadStopLevel >= 11)
         {
-            o_l2_is_scannable = 0;
-            o_l3_is_scannable = 0;
+            //Set all attributes to 0, because in stop 11 nad greater.. can't
+            //access any quad chiplet and it's units
+            for (auto cnt = 0; cnt < MAX_L2_PER_QUAD; ++cnt)
+            {
+                o_l2_is_scomable[cnt]  = 0;
+                o_l2_is_scannable[cnt] = 0;
+            }
+
+            for (auto cnt = 0; cnt < MAX_L3_PER_QUAD; ++cnt)
+            {
+                o_l3_is_scomable[cnt]  = 0;
+                o_l3_is_scannable[cnt] = 0;
+            }
         }
         else
         {
             //Read clock status to confirm stop state history is accurate
             //If we trust the stop state history, this could be removed to save on code size
             //Compare Hardware status vs stop state status. If there is a mismatch the HW value overrides the stop state
+            FAPI_TRY(p9_query_cache_clock_state(i_target, o_l2_is_scomable, o_l3_is_scomable), "Error querying clock state");
+        }
+    }
+    else
+    {
+        //Read clock state if stop state history register doesn't have stop
+        //gated info
+        FAPI_TRY(p9_query_cache_clock_state(i_target, o_l2_is_scomable, o_l3_is_scomable), "Error querying clock state");
+    }
 
-            FAPI_TRY(fapi2::getScom(i_target, EQ_CLOCK_STAT_SL, l_data64), "Error reading data from EQ_CLOCK_STAT_SL");
-
-            FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_target, l_chpltNumber),
-                     "Error: Failed to get the position of the EX:0x%08X", i_target);
-            l_exPos = l_chpltNumber % 2;
+fapi_try_exit:
+    FAPI_INF("< p9_query_cache_access_state...");
+    return fapi2::current_err;
+}
 
-            l_is_scomable = !l_data64.getBit(eq_clk_l2_pos[l_exPos]);
+fapi2::ReturnCode
+p9_query_cache_clock_state(
+    const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target,
+    bool o_l2_is_scomable[MAX_L2_PER_QUAD],
+    bool o_l3_is_scomable[MAX_L3_PER_QUAD])
+{
+    FAPI_INF("< p9_query_cache_clock_state...");
+    fapi2::buffer<uint64_t> l_data64;
+    //Read clock status to confirm stop state history is accurate
+    //If we trust the stop state history, this could be removed to save on code size
+    //Compare Hardware status vs stop state status. If there is a mismatch the HW value overrides the stop state
 
-            if (o_l2_is_scomable != l_is_scomable)
-            {
-                FAPI_INF("Clock status didn't match stop state, overriding is_scomable status");
-                o_l2_is_scomable = l_is_scomable;
-            }
+    FAPI_TRY(fapi2::getScom(i_target, EQ_CLOCK_STAT_SL, l_data64), "Error reading data from EQ_CLOCK_STAT_SL");
 
-            l_is_scomable = !l_data64.getBit(eq_clk_l3_pos[l_exPos]);
+    // Need to look for both l20(ex0),l21(ex1) and l30,l31 bits info
+    for (auto l_l2Pos = 0; l_l2Pos < MAX_L2_PER_QUAD; l_l2Pos++)
+    {
+        o_l2_is_scomable[l_l2Pos] = !l_data64.getBit(eq_clk_l2_pos[l_l2Pos]);
+    }
 
-            if (o_l3_is_scomable != l_is_scomable)
-            {
-                FAPI_INF("Clock status didn't match stop state, overriding is_scomable status");
-                o_l3_is_scomable = l_is_scomable;
-            }
-        }
+    for (auto l_l3Pos = 0; l_l3Pos < MAX_L3_PER_QUAD; l_l3Pos++)
+    {
+        o_l3_is_scomable[l_l3Pos] = !l_data64.getBit(eq_clk_l3_pos[l_l3Pos]);
     }
 
 fapi_try_exit:
-    FAPI_INF("< p9_query_cache_access_state...");
+    FAPI_INF("< p9_query_cache_clock_state...");
     return fapi2::current_err;
+
 }
diff --git a/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.H b/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.H
index 451ee4c9f..3112f18aa 100644
--- a/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.H
+++ b/src/import/chips/p9/procedures/hwp/pm/p9_query_cache_access_state.H
@@ -48,14 +48,15 @@
 //------------------------------------------------------------------------------
 // Constant definitions
 //------------------------------------------------------------------------------
-
+#define MAX_L2_PER_QUAD 2 //shared by the core/ex
+#define MAX_L3_PER_QUAD 2 //shared by the core/ex
 // function pointer typedef definition for HWP call support
 typedef fapi2::ReturnCode (*p9_query_cache_access_state_FP_t) (
     const fapi2::Target<fapi2::TARGET_TYPE_EQ>&,
-    bool&,
-    bool&,
-    bool&,
-    bool&);
+    bool*,
+    bool*,
+    bool*,
+    bool*);
 
 extern "C"
 {
@@ -68,8 +69,10 @@ extern "C"
 ///
 /// @param[in] i_target        EX target
 ///
-/// @param[out] o_l2_is_scomable  L2 cache has clocks running and is scomable
-/// @param[out[ o_l2_is_scannable L2 cache is powered up and has valid latch state
+/// @param[out] o_l2_is_scomable[MAX_L2_PER_QUAD]
+//                           L2 cache has clocks running and is scomable
+/// @param[out[ o_l2_is_scannable[MAX_L2_PER_QUAD]
+//                           L2 cache is powered up and has valid latch state
 /// @param[out] o_l3_is_scomable  L3 cache has clocks running and is scomable
 /// @param[out[ o_l2_is_scannable L3 cache is powered up and has valid latch state
 ///
@@ -78,10 +81,26 @@ extern "C"
     fapi2::ReturnCode
     p9_query_cache_access_state(
         const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target,
-        bool& o_l2_is_scomable,
-        bool& o_l2_is_scannable,
-        bool& o_l3_is_scomable,
-        bool& o_l3_is_scannable);
+        bool  o_l2_is_scomable[MAX_L2_PER_QUAD],
+        bool  o_l2_is_scannable[MAX_L2_PER_QUAD],
+        bool  o_l3_is_scomable[MAX_L3_PER_QUAD],
+        bool  o_l3_is_scannable[MAX_L3_PER_QUAD]);
+
+/// @brief Check the stop level for the EX caches and sets boolean scomable parameters
+///
+/// @param[in] i_target        EX target
+///
+/// @param[out] o_l2_is_scomable[MAX_L2_PER_QUAD]
+//                           L2 cache has clocks running and is scomable
+/// @param[out] o_l3_is_scomable  L3 cache has clocks running and is scomable
+///
+/// @return FAPI2_RC_SUCCESS if success, else error code.
+
+    fapi2::ReturnCode
+    p9_query_cache_clock_state(
+        const fapi2::Target<fapi2::TARGET_TYPE_EQ>& i_target,
+        bool  o_l2_is_scomable[MAX_L2_PER_QUAD],
+        bool  o_l3_is_scomable[MAX_L3_PER_QUAD]);
 
 } // extern "C"