Forcibly clear all previous wakeups when the PM Complex starts

The power management logic was designed with the assumption
that the external callers would manage the state machine such
that the order of operations is always:
  load->start->reset->load->start->reset->etc.  However, we
have discovered edge cases where a reset can happen before the
first load or after a previous reset.  In either case we will
end up with our wakeup counter being off by one and thus never
releasing wakeup.

The solution is to add logic to the start path that will
explicitly clear the counter out completely, regardless of its
current value.  This will ensure that wakeup is released once
the complex is alive.

Change-Id: Ifb436335090ff910b9ee0f083d9a550b81833287
CQ: SW442639
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/65518
Reviewed-by: Matt Derksen <mderkse1@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

8 files changed, 182 insertions, 34 deletions

diff --git a/src/include/runtime/interface.h b/src/include/runtime/interface.h
index 21e596ab5..ae922d23c 100644
--- a/src/include/runtime/interface.h
+++ b/src/include/runtime/interface.h
@@ -127,9 +127,12 @@ enum HbrtRcPiberr_t
  *  Wakeup mode for the wakeup() interface
  *      HBRT_WKUP_FORCE_AWAKE: force a core awake
  *      HBRT_WKUP_CLEAR_FORCE: clear a previous force
+ *      HBRT_WKUP_CLEAR_FORCE_COMPLETELY: clear all previous forces, regardless
+ *                                        of internal counts
  */
 #define HBRT_WKUP_FORCE_AWAKE  0
 #define HBRT_WKUP_CLEAR_FORCE  1
+#define HBRT_WKUP_CLEAR_FORCE_COMPLETELY  2
 
 /**
  *  Chip ID types included in the chip_id / proc_id / core_id
@@ -167,6 +170,7 @@ enum HbrtRcPiberr_t
 #define HBRT_CAPS_SET1_OPAL    1
 #define HBRT_CAPS_OPAL_HAS_XSCOM_RC     (1ul << 0)
 #define HBRT_CAPS_OPAL_HAS_WAKEUP       (1ul << 1)
+#define HBRT_CAPS_OPAL_HAS_WAKEUP_CLEAR (1ul << 2)
 
 /* PHYP fixes */
 #define HBRT_CAPS_SET2_PHYP    2
diff --git a/src/include/usr/scom/scomreasoncodes.H b/src/include/usr/scom/scomreasoncodes.H
index e8d5a7e6b..5c39abdeb 100644
--- a/src/include/usr/scom/scomreasoncodes.H
+++ b/src/include/usr/scom/scomreasoncodes.H
@@ -79,6 +79,8 @@ namespace SCOM
         SCOM_SENSITIVE_REG_READ_FAIL   = SCOM_COMP_ID | 0x1A,
         SCOM_BAD_TARGET                = SCOM_COMP_ID | 0x1B,
         SCOM_CACHE_SEQ_ERROR           = SCOM_COMP_ID | 0x1C,
+        SCOM_INVALID_WAKEUP_PARM       = SCOM_COMP_ID | 0x1D,
+        SCOM_UNEXPECTED_FORCE_WAKEUP   = SCOM_COMP_ID | 0x1E,
     };
 
     enum UserDetailsTypes
diff --git a/src/include/usr/scom/wakeup.H b/src/include/usr/scom/wakeup.H
new file mode 100644
index 000000000..96e659197
--- /dev/null
+++ b/src/include/usr/scom/wakeup.H
@@ -0,0 +1,56 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/include/usr/scom/wakeup.H $                               */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2018                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+/**
+ * Contains functions and constants related to handling special wakeup.
+ */
+
+namespace WAKEUP
+{
+
+/**
+ * Parameters for wakeup assert/release
+ */
+enum HandleOptions_t
+{
+    ENABLE,
+    DISABLE,
+    FORCE_DISABLE
+};
+
+/**
+ * @brief This function handles the enable and disable of the special
+ *        wakeup that allows scom operations on idle cores.
+ *
+ * @param[in] i_target   - EQ/EX/Core target
+ * @param[in] i_enable   - set or clear or clear all of the wakeups
+ *
+ * @return errlHndl_t    - error handle
+ */
+errlHndl_t handleSpecialWakeup(TARGETING::Target* i_target,
+                               HandleOptions_t i_enable);
+
+
+
+}
diff --git a/src/usr/fapi2/plat_utils.C b/src/usr/fapi2/plat_utils.C
index 2825ffaa5..ee1fde715 100644
--- a/src/usr/fapi2/plat_utils.C
+++ b/src/usr/fapi2/plat_utils.C
@@ -47,8 +47,7 @@
 #include <p9_tor.H>
 #include <p9_scan_compression.H>
 #include <cen_ringId.H>
-
-#include "handleSpecialWakeup.H"
+#include <scom/wakeup.H>
 
 //******************************************************************************
 // Trace descriptors
@@ -1500,7 +1499,13 @@ fapi2::ReturnCode platSpecialWakeup(const Target<TARGET_TYPE_ALL>& i_target,
         reinterpret_cast<TARGETING::Target*>(i_target.get());
     FAPI_INF("platSpecialWakeup : HUID=%.8X, enable=%d", TARGETING::get_huid(l_target), i_enable);
 
-    errlHndl_t err_SW = handleSpecialWakeup(l_target,i_enable);
+    WAKEUP::HandleOptions_t l_option = WAKEUP::DISABLE;
+    if( i_enable )
+    {
+        l_option = WAKEUP::ENABLE;
+    }
+
+    errlHndl_t err_SW = WAKEUP::handleSpecialWakeup(l_target,l_option);
     if(err_SW)
     {
         fapi_rc.setPlatDataPtr(reinterpret_cast<void *>(err_SW));
diff --git a/src/usr/isteps/pm/runtime/makefile b/src/usr/isteps/pm/runtime/makefile
index 2a98e0064..ecffa4f37 100644
--- a/src/usr/isteps/pm/runtime/makefile
+++ b/src/usr/isteps/pm/runtime/makefile
@@ -31,8 +31,6 @@ VPATH += ../
 
 MODULE = pm_rt
 
-EXTRAINCDIR += ${ROOTPATH}/src/usr/scom/
-
 ## Objects unique to HBRT
 OBJS += rt_pm.o
 
diff --git a/src/usr/isteps/pm/runtime/rt_pm.C b/src/usr/isteps/pm/runtime/rt_pm.C
index a3087e504..ef861563d 100644
--- a/src/usr/isteps/pm/runtime/rt_pm.C
+++ b/src/usr/isteps/pm/runtime/rt_pm.C
@@ -48,7 +48,7 @@
 #include    <targeting/common/targetservice.H>
 
 #include <scom/scomif.H>
-#include "handleSpecialWakeup.H"
+#include <scom/wakeup.H>
 
 using namespace TARGETING;
 using namespace RUNTIME;
@@ -240,6 +240,21 @@ namespace RTPM
                 break;
             }
 
+            // The PM Complex is now live, ensure that there are no
+            //  lingering special wakeups enabled
+            l_err = WAKEUP::handleSpecialWakeup( proc_target,
+                                                 WAKEUP::FORCE_DISABLE );
+            if( l_err )
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           "Error disabling wakeup on %.8X",
+                           TARGETING::get_huid(proc_target) );
+                //Just commit the log as informational and keep going
+                l_err->setSev(ERRL_SEV_INFORMATIONAL);
+                l_err->collectTrace(ISTEP_COMP_NAME,1024);
+                errlCommit( l_err, RUNTIME_COMP_ID );
+                l_err = nullptr;
+            }
         } while(0);
 
         if ( l_err )
@@ -252,6 +267,7 @@ namespace RTPM
     }
 
 
+
     /**
      *  @brief Reset OCC/HCODE on the specified chip
      *  @param[in]  i_chip              Processor Chip ID
@@ -366,7 +382,7 @@ namespace RTPM
             }
 
             // Enable special wakeup
-            l_err = handleSpecialWakeup(i_target,true);
+            l_err = handleSpecialWakeup(i_target,WAKEUP::ENABLE);
             if(l_err)
             {
                 TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
@@ -485,7 +501,7 @@ namespace RTPM
             } // END else if (g_hostInterfaces->firmware_request != nullptr)
 
             // Disable special wakeup
-            l_err = handleSpecialWakeup(i_target,false);
+            l_err = handleSpecialWakeup(i_target,WAKEUP::DISABLE);
             if(l_err)
             {
                 TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
diff --git a/src/usr/scom/handleSpecialWakeup.C b/src/usr/scom/handleSpecialWakeup.C
index 65be26b03..11fbb94b1 100644
--- a/src/usr/scom/handleSpecialWakeup.C
+++ b/src/usr/scom/handleSpecialWakeup.C
@@ -43,9 +43,7 @@
 
 #include <fapi2/plat_hwp_invoker.H>
 #include <p9_cpu_special_wakeup.H>
-
-extern "C"
-{
+#include <scom/wakeup.H>
 
 // Trace definition
 extern trace_desc_t* g_trac_scom;
@@ -53,6 +51,9 @@ extern trace_desc_t* g_trac_scom;
 using namespace TARGETING;
 using namespace SCOM;
 
+namespace WAKEUP
+{
+
 /**
  * @brief Check to see which type of wakeup we need to use
  * @return  true: Use Host wakeup interface;
@@ -98,9 +99,10 @@ bool useHypWakeup( void )
 /**
  * @brief Use the Host/Hyp interface to control special wakeup
  * @param i_target  Input target core/ex/eq/etc
- * @param i_enable  Turn wakeup on or off
+ * @param[in] i_enable   - set or clear or clear all of the wakeups
  */
-errlHndl_t callWakeupHyp(TARGETING::Target* i_target, bool i_enable)
+errlHndl_t callWakeupHyp(TARGETING::Target* i_target,
+                         HandleOptions_t i_enable)
 {
     errlHndl_t l_errl = NULL;
 
@@ -162,14 +164,48 @@ errlHndl_t callWakeupHyp(TARGETING::Target* i_target, bool i_enable)
         }
 
         uint32_t mode;
-        if(i_enable)
+        if(i_enable == ENABLE)
         {
             mode = HBRT_WKUP_FORCE_AWAKE;
         }
-        else
+        else if(i_enable == DISABLE)
         {
             mode = HBRT_WKUP_CLEAR_FORCE;
         }
+        else if(i_enable == FORCE_DISABLE)
+        {
+            mode = HBRT_WKUP_CLEAR_FORCE_COMPLETELY;
+        }
+        else
+        {
+            /*@
+             * @errortype
+             * @moduleid         SCOM_HANDLE_SPECIAL_WAKEUP
+             * @reasoncode       SCOM_INVALID_WAKEUP_PARM
+             * @userdata1        Wakeup Argument
+             * @userdata2        Input Target
+             * @devdesc          Invalid mode parm for wakeup operation.
+             */
+            l_errl = new ERRORLOG::ErrlEntry(
+                                        ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                                        SCOM_HANDLE_SPECIAL_WAKEUP,
+                                        SCOM_INVALID_WAKEUP_PARM,
+                                        i_enable,
+                                        TARGETING::get_huid(i_target),
+                                        ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
+            break;
+        }
+
+        // PHYP supports the complete clear but OPAL might not yet
+        if( (mode == HBRT_WKUP_CLEAR_FORCE_COMPLETELY)
+            && !TARGETING::is_phyp_load()
+            && !(g_hostInterfaces->
+                 get_interface_capabilities(HBRT_CAPS_SET1_OPAL)
+                 & HBRT_CAPS_OPAL_HAS_WAKEUP_CLEAR) )
+        {
+            TRACFCOMP( g_trac_scom, "No support for forced clear, skipping" );
+            break;
+        }
 
         // Do the special wakeup
         int l_rc = g_hostInterfaces->wakeup(rtTargetId,mode);
@@ -221,9 +257,10 @@ errlHndl_t callWakeupHyp(TARGETING::Target* i_target, bool i_enable)
 /**
  * @brief Call the wakeup HWP to control special wakeup
  * @param i_target  Input target core/ex/eq/etc
- * @param i_enable  Turn wakeup on or off
+ * @param[in] i_enable   - set or clear or clear all of the wakeups
  */
-errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
+errlHndl_t callWakeupHwp(TARGETING::Target* i_target,
+                         HandleOptions_t i_enable)
 {
     errlHndl_t l_errl = NULL;
     fapi2::ReturnCode l_rc;
@@ -255,7 +292,7 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
     // Assume HBRT is single-threaded, so no issues with concurrency
     uint32_t l_count = (i_target)->getAttr<ATTR_SPCWKUP_COUNT>();
 
-    if((l_count==0) && !i_enable)
+    if((l_count==0) && (i_enable==WAKEUP::DISABLE))
     {
         TRACFCOMP( g_trac_scom,ERR_MRK
             "Disabling special wakeup on target with SPCWKUP_COUNT=0");
@@ -266,7 +303,7 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
          * @reasoncode       SCOM_SPCWKUP_COUNT_ERR
          * @userdata1        Target HUID
          * @userdata2[0:31]  Wakeup Enable
-         * @userdata2[32:63] Wakeup Count
+         * @userdata2[32:63] Wakeup Count (ATTR_SPCWKUP_COUNT)
          * @devdesc          Disabling special wakeup when not enabled.
          */
         l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_INFORMATIONAL,
@@ -274,17 +311,42 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
                                          SCOM_SPCWKUP_COUNT_ERR,
                                          get_huid(i_target),
                                          TWO_UINT32_TO_UINT64(
-                                                    i_enable, l_count));
+                                                    i_enable, l_count),
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
 
-        l_errl->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
-                                    HWAS::SRCI_PRIORITY_LOW);
+        errlCommit( l_errl, RUNTIME_COMP_ID );
+    }
+
+    if( (l_count>0) && (i_enable==WAKEUP::FORCE_DISABLE) )
+    {
+        TRACFCOMP( g_trac_scom,ERR_MRK
+                   "Attempting to force disable special wakeup on %.8X with SPCWKUP_COUNT=%d",
+                   TARGETING::get_huid(i_target), l_count);
+        /*@
+         * @errortype
+         * @moduleid         SCOM_CALL_WAKEUP_HWP
+         * @reasoncode       SCOM_UNEXPECTED_FORCE_WAKEUP
+         * @userdata1        Target HUID
+         * @userdata2[0:31]  Wakeup Enable
+         * @userdata2[32:63] Wakeup Count (ATTR_SPCWKUP_COUNT)
+         * @devdesc          Unexpectedly forcing wakeup off when the counter
+         *                   is non-zero, implies a bug in the code flow.
+         */
+        l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                                         SCOM_CALL_WAKEUP_HWP,
+                                         SCOM_UNEXPECTED_FORCE_WAKEUP,
+                                         get_huid(i_target),
+                                         TWO_UINT32_TO_UINT64(
+                                                    i_enable, l_count),
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT );
 
         errlCommit( l_errl, RUNTIME_COMP_ID );
     }
 
-    // Only call the HWP if 0-->1 or 1-->0
-    if( ((l_count==0) &&  i_enable) ||
-        ((l_count==1) && !i_enable) )
+    // Only call the HWP if 0-->1 or 1-->0 or if it is a force
+    if( ((l_count==0) && (i_enable==WAKEUP::ENABLE)) ||
+        ((l_count==1) && (i_enable==WAKEUP::DISABLE)) ||
+        ((l_count>1) && (i_enable==WAKEUP::FORCE_DISABLE)) )
     {
         // NOTE Regarding the entity type passed to the HWP:
         // There are 3 independent registers used to trigger a
@@ -302,13 +364,13 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
             l_spcwkupSrc = p9specialWakeup::HOST;
         }
 
-        if(i_enable)
+        if(i_enable==WAKEUP::DISABLE)
         {
-            l_spcwkupType = p9specialWakeup::SPCWKUP_ENABLE;
+            l_spcwkupType = p9specialWakeup::SPCWKUP_DISABLE;
         }
         else
         {
-            l_spcwkupType = p9specialWakeup::SPCWKUP_DISABLE;
+            l_spcwkupType = p9specialWakeup::SPCWKUP_ENABLE;
         }
 
         if(l_type == TARGETING::TYPE_EQ)
@@ -363,14 +425,18 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
     // Update the counter
     if(!l_errl)
     {
-        if(i_enable)
+        if(i_enable == ENABLE)
         {
             l_count++;
         }
-        else
+        else if(i_enable == DISABLE)
         {
             l_count--;
         }
+        else if(i_enable == FORCE_DISABLE)
+        {
+            l_count = 0;
+        }
         i_target->setAttr<ATTR_SPCWKUP_COUNT>(l_count);
     }
 
@@ -381,7 +447,8 @@ errlHndl_t callWakeupHwp(TARGETING::Target* i_target, bool i_enable)
 /**
  * @brief Enable and disable special wakeup for SCOM operations
  */
-errlHndl_t handleSpecialWakeup(TARGETING::Target* i_target, bool i_enable)
+errlHndl_t handleSpecialWakeup(TARGETING::Target* i_target,
+                               HandleOptions_t i_enable)
 {
     errlHndl_t l_errl = NULL;
 
diff --git a/src/usr/scom/scomtrans.C b/src/usr/scom/scomtrans.C
index 06cf404fc..f100a5e3c 100644
--- a/src/usr/scom/scomtrans.C
+++ b/src/usr/scom/scomtrans.C
@@ -56,7 +56,7 @@
 #include <hw_access_def.H>
 
 #if __HOSTBOOT_RUNTIME
-  #include "handleSpecialWakeup.H"
+  #include <scom/wakeup.H>
 #endif
 
 // Trace definition
@@ -208,7 +208,7 @@ errlHndl_t startScomProcess(DeviceFW::OperationType i_opType,
             TRACDCOMP(g_trac_scom,"Special wakeup required, starting now..");
             g_wakeupInProgress = true;
 
-            l_err = handleSpecialWakeup(i_target,true);
+            l_err = WAKEUP::handleSpecialWakeup(i_target,WAKEUP::ENABLE);
             if(l_err)
             {
                 TRACFCOMP(g_trac_scom, "startScomProcess: "
@@ -242,7 +242,7 @@ errlHndl_t startScomProcess(DeviceFW::OperationType i_opType,
             g_wakeupInProgress = true;
             errlHndl_t l_errSW = NULL;
 
-            l_errSW = handleSpecialWakeup(i_target,false);
+            l_errSW = WAKEUP::handleSpecialWakeup(i_target,WAKEUP::DISABLE);
 
             if(l_err != NULL && l_errSW)
             {