Ensure all hbmutex attributes get re-initialized on MPIPL

We saw a bug where we were getting hung up on one of the i2c engine
mutexes on MPIPL. We still haven't determined WHY we are seeing the
mutex locked after an MPIPL but just to be safe we are now looping
through all of the targets and checking each attribute to see if it
is of type hbmutex. If it is then we will call mutex_init on the mutex
to reset it so the mutex is ready to use.

Change-Id: Ie05f7f49c4599c7cfc5d33c43d79cb977bb0cd13
CQ: SW420894
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/56136
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Roland Veloz <rveloz@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

6 files changed, 215 insertions, 4 deletions

diff --git a/src/include/usr/targeting/common/targetservice.H b/src/include/usr/targeting/common/targetservice.H
index 056dfb9a5..2c8133ae8 100644
--- a/src/include/usr/targeting/common/targetservice.H
+++ b/src/include/usr/targeting/common/targetservice.H
@@ -776,8 +776,24 @@ class TargetService
                                      AttrRP *i_attrRP,
                                      ATTRIBUTE_ID* &o_pAttrId,
                                      AbstractPointer<void>* &o_ppAttrAddr);
-
-
+#ifdef __HOSTBOOT_MODULE
+        /**
+         *  @brief Reset all hostboot mutex attributes
+         *
+         *  @par Detailed Description:
+         *  Iterates through all targets and check each of their attributes to
+         *  see if they are of type hbmutex. If we find a mutex attribute we will
+         *  run mutex_init on it to reset it.
+         *
+         *  @pre TargetService must be initialized.
+         *       AttrRp must be initialized
+         *
+         *  @post All hb mutex attributes on all targets have been reset
+         *
+         *  @return void
+         */
+        void resetMutexAttributes();
+#endif
     private:
 
         /**
diff --git a/src/include/usr/targeting/targplatreasoncodes.H b/src/include/usr/targeting/targplatreasoncodes.H
index ac93bf2b9..b38c0c646 100644
--- a/src/include/usr/targeting/targplatreasoncodes.H
+++ b/src/include/usr/targeting/targplatreasoncodes.H
@@ -61,6 +61,7 @@ enum PlatTargetingModuleId
     TARG_RT_HBRT_UPDATE_PREP                      = 0x85,
     TARG_RT_VALIDATEDATA                          = 0x86,
     TARG_RT_SAVERESTOREATTRS                      = 0x87,
+    TARG_SVC_RESET_MUTEX                          = 0x88,
 };
 
 /**
@@ -80,6 +81,7 @@ enum PlatTargetingReasonCode
     TARG_RT_SECTION_MISMATCH       = TARG_COMP_ID | 0x86,
     TARG_RT_MISSING_ATTR           = TARG_COMP_ID | 0x87,
     TARG_RT_BAD_ATTR_SIZES         = TARG_COMP_ID | 0x88,
+    TARG_SVC_MISSING_ATTR          = TARG_COMP_ID | 0x89,
 };
 
 } // End TARGETING namespace
diff --git a/src/usr/targeting/common/targetservice.C b/src/usr/targeting/common/targetservice.C
index 4489809cf..c6de5a1dd 100644
--- a/src/usr/targeting/common/targetservice.C
+++ b/src/usr/targeting/common/targetservice.C
@@ -53,6 +53,11 @@
 #include <attributetraits.H>
 #include <targeting/common/utilFilter.H>
 
+#ifdef __HOSTBOOT_MODULE
+// Generated
+#include <mutexattributes.H>
+#endif
+
 #undef EXTRA_SANITY_CHECKING
 
 //******************************************************************************
@@ -1569,6 +1574,74 @@ uint32_t TargetService::getTargetAttributes(Target*i_target,
     // Return the number of attributes for this target
     return i_target->iv_attrs;
 }
+#ifdef __HOSTBOOT_MODULE
+void TargetService::resetMutexAttributes()
+{
+    #define TARG_FN "resetMutexAttributes(...)"
+    TARGETING::AttrRP *l_pAttrRP = &TARG_GET_SINGLETON(TARGETING::theAttrRP);
+    ATTRIBUTE_ID* l_pAttrIds = nullptr;
+    AbstractPointer<void>* l_ppAttrAddrs = nullptr;
+    for( auto targ_iter = targetService().begin();
+        targ_iter != targetService().end();
+         targ_iter++)
+    {
+        uint32_t l_attrCount = 0;
+        Target* l_pTarget = *targ_iter;
+        l_attrCount = targetService().getTargetAttributes(l_pTarget, l_pAttrRP,
+                                            l_pAttrIds, l_ppAttrAddrs );
+
+        // Make sure that attributes were found
+        if(l_attrCount == 0)
+        {
+            TRACFCOMP( g_trac_targeting,
+                       "Target 0x%X has no attributes", get_huid(l_pTarget) );
+            // Continue to next target if there were no attributes
+            continue;
+        }
+
+        for ( uint32_t l_attrIndex = 0; l_attrIndex < l_attrCount; l_attrIndex++)
+        {
+            const ATTRIBUTE_ID l_attrId = l_pAttrIds[l_attrIndex];
+            for( const auto mutexId : hbMutexAttrIds)
+            {
+                if(l_attrId == mutexId)
+                {
+                    mutex_t* l_mutex;
+                    if(l_pTarget->_tryGetHbMutexAttr(l_attrId, l_mutex))
+                    {
+                        mutex_init(l_mutex);
+                    }
+                    else
+                    {
+                        /*@
+                        *   @errortype         ERRORLOG::ERRL_SEV_PREDICTIVE
+                        *   @moduleid          TARG_SVC_RESET_MUTEX
+                        *   @reasoncode        TARG_SVC_MISSING_ATTR
+                        *   @userdata1         Attribute Id we attempted to read
+                        *   @userdata2         Huid of target we attempted to read
+                        *
+                        *   @devdesc   For some reason attr IDs in hbMutexAttrIds list
+                        *              are not matching the attribute IDs that target
+                        *              service is seeing. This is causing incorrect matching
+                        *              Make sure mutexattribute.H in genfiles has good values
+                        *
+                        *   @custdesc  Attempted to perform an invalid attribute look up
+                        */
+                        errlHndl_t l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_PREDICTIVE,
+                                                          TARG_SVC_RESET_MUTEX,
+                                                          TARG_SVC_MISSING_ATTR,
+                                                          l_attrId,
+                                                          get_huid(l_pTarget),
+                                                          true); // software error
+                        errlCommit(l_errl, TARG_COMP_ID);
+                    }
+                }
+            }
+        }
+    }
+    #undef TARG_FN
+}
+#endif
 
 #undef TARG_CLASS
 
diff --git a/src/usr/targeting/common/xmltohb/xmltohb.pl b/src/usr/targeting/common/xmltohb/xmltohb.pl
index be87ad0f5..43906fd12 100755
--- a/src/usr/targeting/common/xmltohb/xmltohb.pl
+++ b/src/usr/targeting/common/xmltohb/xmltohb.pl
@@ -266,6 +266,15 @@ if( !($cfgSrcOutputDir =~ "none") )
     writeTargAttrMap($attributes, $targAttrFile);
     close $targAttrFile;
 
+    open(MUTEX_ATTR_FILE, ">$cfgSrcOutputDir"."mutexattributes.H")
+      or croak ("Muxtex Attribute file: \"$cfgSrcOutputDir"
+        . "mutexattributes.H\" could not be opened.");
+    my $mutexFile = *MUTEX_ATTR_FILE;
+    writeMutexFileHeader($mutexFile);
+    writeMutexFileAttrs($attributes,$mutexFile);
+    writeMutexFileFooter($mutexFile);
+    close $mutexFile;
+
 
     open(TRAIT_FILE,">$cfgSrcOutputDir"."attributetraits.H")
       or croak ("Trait file: \"$cfgSrcOutputDir"
@@ -2251,6 +2260,110 @@ sub writeTargAttrMap {
     print $outFile "};\n";
 }
 
+sub writeMutexFileHeader {
+    my($outFile) = @_;
+
+print $outFile <<VERBATIM;
+
+#ifndef TARG_MUTEXATTRIBUTES_H
+#define TARG_MUTEXATTRIBUTES_H
+
+/**
+ *  \@file mutexattributes.H
+ *
+ *  \@brief Array of attributes Ids whose type is hbmutex
+ *
+ *  This header file contains a single array that lists out all of the hbmutex
+ *  attributes. This is used on the MPIPL path to know which attributes we need
+ *  to reset.  This file is autogenerated and should not be altered.
+ */
+
+//******************************************************************************
+// Includes
+//******************************************************************************
+
+// STD
+#include <stdint.h>
+VERBATIM
+}
+
+sub writeMutexFileAttrs {
+    my($attributes,$outFile) = @_;
+
+print $outFile <<VERBATIM;
+
+//******************************************************************************
+// Array
+//******************************************************************************
+
+namespace TARGETING
+{
+
+/**
+ *  \@brief HB Mutex Attribute IDs
+ *
+ *  Array defining all attribute ids found that are of type hbMutex.
+ *  This file is autogenerated and should not be altered.
+ */
+const uint32_t hbMutexAttrIds[] = {
+VERBATIM
+
+    my @mutexAttrIds;
+
+    foreach my $attribute (@{$attributes->{attribute}})
+    {
+        #check if hbmutex tag is present
+        #check that attr is readable/writeable
+        if(   (exists $attribute->{simpleType})
+                && (exists $attribute->{simpleType}->{hbmutex})
+                && (exists $attribute->{readable})
+                && (exists $attribute->{writeable}))
+        {
+            push @mutexAttrIds, $attribute->{id};
+        }
+    }
+
+    # variables that can be used for writing the enums to the file
+    my $attrId;
+    my $hexVal;
+
+    # Format below intentionally > 80 chars for clarity
+    format ATTRMUTEXFORMAT =
+                                  @>>>>>>>>>>
+    $hexVal .","
+.
+    select($outFile);
+    $~ = 'ATTRMUTEXFORMAT';
+
+    my $attrIdEnum = getAttributeIdEnumeration($attributes);
+
+    foreach my $enumerator (@{$attrIdEnum->{enumerator}})
+    {
+        $hexVal = $enumerator->{value};
+        $attrId = $enumerator->{name};
+        foreach my $mutexAttrId (@mutexAttrIds)
+        {
+            if( $mutexAttrId eq $attrId )
+            {
+                write;
+                last;
+            }
+        }
+    }
+
+    print $outFile "};\n\n";
+}
+
+sub writeMutexFileFooter {
+    my($outFile) = @_;
+    print $outFile <<VERBATIM;
+} // End namespace TARGETING
+
+#endif // TARG_MUTEXATTRIBUTES_H
+
+VERBATIM
+}
+
 ################################################################################
 # Writes the trait file header
 ################################################################################
diff --git a/src/usr/targeting/runtime/start_rt.C b/src/usr/targeting/runtime/start_rt.C
index dac7dc137..a60180fd9 100644
--- a/src/usr/targeting/runtime/start_rt.C
+++ b/src/usr/targeting/runtime/start_rt.C
@@ -50,6 +50,9 @@ namespace RT_TARG
         TargetService& l_targetService = targetService();
         l_targetService.init(Singleton<AttrRP>::instance().getNodeCount());
 
+        // Reset hb mutex attributes in case they got stuck in a locked state
+        l_targetService.resetMutexAttributes();
+
         adjustTargeting4Runtime();
 
         // set global that TARG is ready
diff --git a/src/usr/targeting/targetservicestart.C b/src/usr/targeting/targetservicestart.C
index 882c4391a..459f9e777 100755
--- a/src/usr/targeting/targetservicestart.C
+++ b/src/usr/targeting/targetservicestart.C
@@ -191,6 +191,12 @@ static void initTargeting(errlHndl_t& io_pError)
         (void)l_targetService.init();
 
         initializeAttributes(l_targetService, l_isMpipl, l_isIstepMode, l_scratch);
+        //Ensure all mutex attributes are reset on MPIPL
+        if(l_isMpipl)
+        {
+            l_targetService.resetMutexAttributes();
+        }
+
         checkProcessorTargeting(l_targetService);
 
         // Print out top-level model value from loaded targeting values.
@@ -483,8 +489,6 @@ static void initializeAttributes(TargetService& i_targetService,
             {
                 tpm->setAttr<ATTR_HB_TPM_INIT_ATTEMPTED>(0);
                 tpm->setAttr<ATTR_HB_TPM_LOG_MGR_PTR>(0);
-                auto tpmMutex=tpm->getHbMutexAttr<ATTR_HB_TPM_MUTEX>();
-                mutex_init(tpmMutex);
             }
 
             //Assemble list of membuf and zero out some virtual address attributes