Refactor re-init of targeting data during MPIPL/HBRT startup

There are a couple of things we have to do to targets/attrs when we
either transition from Hostboot->HBRT or from HBRT->Hostboot during
an MPIPL. These include resetting all of the mutex attrs so they
are in the unlocked state and updating the pointer which PEER_TARGET
stores as a value to reflect the change in the memory mapping. For
both of these tasks we need to loop through all of the targets.This
refactor allows both these tasks to be done in the same loop.

Change-Id: I23614bba11b2c9eabb97a7ddc7a53937f83dc131
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/58316
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Benjamin J. Weisenbeck <bweisenb@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 274 insertions, 227 deletions

diff --git a/src/usr/targeting/attrrp.C b/src/usr/targeting/attrrp.C
index defe5ceed..9ba04090a 100755
--- a/src/usr/targeting/attrrp.C
+++ b/src/usr/targeting/attrrp.C
@@ -101,6 +101,14 @@ namespace TARGETING
                 break;
             }
 
+            // Now that the VMM blocks have been created we must set
+            // the appropriate R/W permissions
+            l_errl = this->editPagePermissions(ALL_SECTION_TYPES, DEFAULT_PERMISSIONS);
+            if (l_errl)
+            {
+                break;
+            }
+
             // Spawn daemon thread.
             task_create(&AttrRP::startMsgServiceTask, this);
 
@@ -649,69 +657,113 @@ namespace TARGETING
 
     }
 
-    errlHndl_t AttrRP::createVmmSections()
+    errlHndl_t AttrRP::editPagePermissions(uint8_t i_type, uint32_t i_permission)
     {
         errlHndl_t l_errl = NULL;
-
+        int rc;
+        uint32_t l_perm = i_permission;
         do
         {
-            // Allocate message queue for VMM requests.
-            iv_msgQ = msg_q_create();
-
-            // register it so it can be discovered by istep 21 and thus allow
-            // secure runtime preparation of persistent r/w attributes
-            int rc = msg_q_register(iv_msgQ, ATTRRP_MSG_Q);
-
-            assert(rc == 0, "Bug! Unable to register message queue");
-
             // Create VMM block for each section, assign permissions.
             for (size_t i = 0; i < iv_sectionCount; ++i)
             {
-                uint64_t l_perm = 0;
-                switch(iv_sections[i].type)
+                if(i_permission == DEFAULT_PERMISSIONS)
                 {
-                    case SECTION_TYPE_PNOR_RO:
-                        l_perm = READ_ONLY;
-                        break;
-
-                    case SECTION_TYPE_PNOR_RW:
-                        l_perm = WRITABLE | WRITE_TRACKED;
-                        break;
+                    switch(iv_sections[i].type)
+                    {
+                        case SECTION_TYPE_PNOR_RO:
+                            l_perm = READ_ONLY;
+                            break;
 
-                    case SECTION_TYPE_HEAP_PNOR_INIT:
-                        l_perm = WRITABLE;
-                        break;
+                        case SECTION_TYPE_PNOR_RW:
+                            l_perm = WRITABLE | WRITE_TRACKED;
+                            break;
 
-                    case SECTION_TYPE_HEAP_ZERO_INIT:
-                    case SECTION_TYPE_HB_HEAP_ZERO_INIT:
-                        l_perm = WRITABLE | ALLOCATE_FROM_ZERO;
-                        break;
+                        case SECTION_TYPE_HEAP_PNOR_INIT:
+                            l_perm = WRITABLE;
+                            break;
 
-                    default:
+                        case SECTION_TYPE_HEAP_ZERO_INIT:
+                        case SECTION_TYPE_HB_HEAP_ZERO_INIT:
+                            l_perm = WRITABLE | ALLOCATE_FROM_ZERO;
+                            break;
 
-                        /*@
-                         *   @errortype  ERRORLOG::ERRL_SEV_UNRECOVERABLE
-                         *   @moduleid   TARG_CREATE_VMM_SECTIONS
-                         *   @reasoncode TARG_RC_UNHANDLED_ATTR_SEC_TYPE
-                         *   @userdata1  Section type
-                         *
-                         *   @devdesc    Found unhandled attribute section type
-                         *   @custdesc   FW error, unexpected Attribute section type
-                         */
-                        const bool hbSwError = true;
-                        l_errl = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
-                                               TARG_CREATE_VMM_SECTIONS,
-                                               TARG_RC_UNHANDLED_ATTR_SEC_TYPE,
-                                               iv_sections[i].type,
-                                               0, hbSwError);
-                        break;
+                        default:
+
+                            /*@
+                            *   @errortype  ERRORLOG::ERRL_SEV_UNRECOVERABLE
+                            *   @moduleid   TARG_EDIT_PAGE_PERMISSIONS
+                            *   @reasoncode TARG_RC_UNHANDLED_ATTR_SEC_TYPE
+                            *   @userdata1  Section type
+                            *
+                            *   @devdesc    Found unhandled attribute section type
+                            *   @custdesc   FW error, unexpected Attribute section type
+                            */
+                            const bool hbSwError = true;
+                            l_errl = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                                TARG_EDIT_PAGE_PERMISSIONS,
+                                                TARG_RC_UNHANDLED_ATTR_SEC_TYPE,
+                                                iv_sections[i].type,
+                                                0, hbSwError);
+                                                break;
+                    }
+                }
+                if( i_type == ALL_SECTION_TYPES || i_type == iv_sections[i].type)
+                {
+                    rc = mm_set_permission(reinterpret_cast<void*>(
+                                    iv_sections[i].vmmAddress),
+                                    iv_sections[i].size,
+                                    l_perm);
                 }
 
-                if(l_errl)
+                if (rc)
                 {
+                    /*@
+                        *   @errortype         ERRORLOG::ERRL_SEV_UNRECOVERABLE
+                        *   @moduleid          TARG_EDIT_PAGE_PERMISSIONS
+                        *   @reasoncode        TARG_RC_MM_PERM_FAIL
+                        *   @userdata1         vAddress attempting to allocate.
+                        *   @userdata2         (kernel-rc << 32) | (Permissions)
+                        *
+                        *   @devdesc   While attempting to set permissions on
+                        *              a virtual memory block for an attribute
+                        *              section, the kernel returned an error.
+                        *
+                        *   @custdesc  Kernel failed to set permissions on
+                        *              virtual memory block
+                        */
+                    const bool hbSwError = true;
+                    l_errl = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                            TARG_EDIT_PAGE_PERMISSIONS,
+                                            TARG_RC_MM_PERM_FAIL,
+                                            iv_sections[i].vmmAddress,
+                                            TWO_UINT32_TO_UINT64(rc, l_perm),
+                                            hbSwError);
                     break;
                 }
+            }
+        } while(0);
+        return l_errl;
+    }
 
+    errlHndl_t AttrRP::createVmmSections()
+    {
+        errlHndl_t l_errl = NULL;
+
+        do
+        {
+            // Allocate message queue for VMM requests.
+            iv_msgQ = msg_q_create();
+
+            // register it so it can be discovered by istep 21 and thus allow
+            // secure runtime preparation of persistent r/w attributes
+            int rc = msg_q_register(iv_msgQ, ATTRRP_MSG_Q);
+
+            assert(rc == 0, "Bug! Unable to register message queue");
+
+            // Create VMM block for each section, assign permissions.
+            for (size_t i = 0; i < iv_sectionCount; ++i)
+            {
                 int rc = 0;
                 msg_q_t l_msgQ = iv_msgQ;
 
@@ -750,8 +802,6 @@ namespace TARGETING
 
                 if(iv_sections[i].type == SECTION_TYPE_PNOR_RW)
                 {
-                    // TODO RTC:164480 For MPIPL we need to map the RW section
-                    // in real memory to virtual address of the section in PNOR
                     /*
                      * Register this memory range to be FLUSHed during
                      * a shutdown.
@@ -760,37 +810,6 @@ namespace TARGETING
                         reinterpret_cast<void*>(iv_sections[i].vmmAddress),
                         iv_sections[i].size,ATTR_PRIORITY);
                 }
-
-                rc = mm_set_permission(reinterpret_cast<void*>(
-                                       iv_sections[i].vmmAddress),
-                                       iv_sections[i].size,
-                                       l_perm);
-
-                if (rc)
-                {
-                    /*@
-                     *   @errortype         ERRORLOG::ERRL_SEV_UNRECOVERABLE
-                     *   @moduleid          TARG_CREATE_VMM_SECTIONS
-                     *   @reasoncode        TARG_RC_MM_PERM_FAIL
-                     *   @userdata1         vAddress attempting to allocate.
-                     *   @userdata2         (kernel-rc << 32) | (Permissions)
-                     *
-                     *   @devdesc   While attempting to set permissions on
-                     *              a virtual memory block for an attribute
-                     *              section, the kernel returned an error.
-                     *
-                     *   @custdesc  Kernel failed to set permissions on
-                     *              virtual memory block
-                     */
-                    const bool hbSwError = true;
-                    l_errl = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
-                                           TARG_CREATE_VMM_SECTIONS,
-                                           TARG_RC_MM_PERM_FAIL,
-                                           iv_sections[i].vmmAddress,
-                                           TWO_UINT32_TO_UINT64(rc, l_perm),
-                                           hbSwError);
-                    break;
-                }
             } // End iteration through each section
 
             if(l_errl)
diff --git a/src/usr/targeting/common/targetservice.C b/src/usr/targeting/common/targetservice.C
index a92423a92..709e57560 100644
--- a/src/usr/targeting/common/targetservice.C
+++ b/src/usr/targeting/common/targetservice.C
@@ -54,6 +54,8 @@
 #include <targeting/common/utilFilter.H>
 
 #ifdef __HOSTBOOT_MODULE
+// System
+#include <sys/mm.h>
 // Generated
 
 #include <arch/pirformat.H>
@@ -1703,72 +1705,143 @@ uint32_t TargetService::getTargetAttributes(Target*i_target,
     // Return the number of attributes for this target
     return i_target->iv_attrs;
 }
+
 #ifdef __HOSTBOOT_MODULE
-void TargetService::resetMutexAttributes()
+
+#ifndef __HOSTBOOT_RUNTIME
+errlHndl_t TargetService::modifyReadOnlyPagePermissions(bool i_allowWrites)
 {
-    #define TARG_FN "resetMutexAttributes(...)"
+    #define TARG_FN "modifyReadOnlyPagePermissions(...)"
+    TARG_ENTER();
+    errlHndl_t l_errl = NULL;
     TARGETING::AttrRP *l_pAttrRP = &TARG_GET_SINGLETON(TARGETING::theAttrRP);
-    ATTRIBUTE_ID* l_pAttrIds = nullptr;
-    AbstractPointer<void>* l_ppAttrAddrs = nullptr;
-    for( auto targ_iter = targetService().begin();
-        targ_iter != targetService().end();
-         targ_iter++)
+    if(i_allowWrites)
     {
-        uint32_t l_attrCount = 0;
-        Target* l_pTarget = *targ_iter;
-        l_attrCount = targetService().getTargetAttributes(l_pTarget, l_pAttrRP,
-                                            l_pAttrIds, l_ppAttrAddrs );
+        l_errl = l_pAttrRP->editPagePermissions(SECTION_TYPE_PNOR_RO, WRITABLE);
+    }
+    else
+    {
+        l_errl = l_pAttrRP->editPagePermissions(SECTION_TYPE_PNOR_RO, READ_ONLY);
+    }
+    TARG_EXIT();
+    #undef TARG_FN
+    return l_errl;
+}
+#endif
+
+bool TargetService::updatePeerTarget(const Target* i_pTarget)
+{
+    #define TARG_FN "updatePeerTarget(...)"
+    // Variable which holds return value
+    bool l_peerTargetUpdated = false;
+    do
+    {
+        TARGETING::Target * l_peer =  static_cast<Target*>(NULL);
+        if(! i_pTarget->tryGetAttr<ATTR_PEER_TARGET>(l_peer))
+        {
+            // This is a normal path, many targets do not have PEERS
+            TRACDCOMP(g_trac_targeting, "No PEER_TARGET found for target 0x%x",
+                      get_huid(i_pTarget));
+            // Skip the rest of the function in this case
+            break;
+        }
+        else
+        {
+            TRACFCOMP(g_trac_targeting, "Initial PEER_TARGET address for HUID 0x%x found to be %p",
+                      get_huid(i_pTarget), l_peer);
+        }
+
+        TARGETING::ATTR_PEER_PATH_type l_peerPath;
+        if(i_pTarget->tryGetAttr<ATTR_PEER_PATH>(l_peerPath))
+        {
+            // If we find a PEER_PATH we need to next look up the PEER_TARGET with toTarget
+            l_peer = targetService().toTarget(l_peerPath);
 
-        // Make sure that attributes were found
-        if(l_attrCount == 0)
+            // Make sure the path resolved to a non-null target
+            if (l_peer != NULL)
+            {
+                l_peerTargetUpdated = i_pTarget->_trySetAttr(ATTR_PEER_TARGET,
+                                                             sizeof(l_peer),
+                                                             &l_peer);
+                TRACFCOMP(g_trac_targeting, "Updated PEER_TARGET address for HUID 0x%x found to be %p",
+                          get_huid(i_pTarget), l_peer);
+            }
+            else
+            {
+                // This is unexpected so make the trace visible, but no need to assert
+                TRACFCOMP(g_trac_targeting,
+                          "PEER_PATH did not resolve to a valid target for the entity path %s",
+                          l_peerPath.toString());
+            }
+        }
+        else
         {
-            TRACFCOMP( g_trac_targeting,
-                       "Target 0x%X has no attributes", get_huid(l_pTarget) );
-            // Continue to next target if there were no attributes
-            continue;
+            // This is unexpected so make the trace visible, but no need to assert
+            TRACFCOMP(g_trac_targeting,
+                      "No PEER_PATH found for target 0x%x which does have a PEER_TARGET attribute",
+                      get_huid(i_pTarget));
         }
+    } while(0);
+
+    return l_peerTargetUpdated;
+    #undef TARG_FN
+}
 
-        for ( uint32_t l_attrIndex = 0; l_attrIndex < l_attrCount; l_attrIndex++)
+uint32_t TargetService::resetMutexAttributes(const Target* i_pTarget)
+{
+    #define TARG_FN "resetMutexAttributes(...)"
+    TARGETING::AttrRP *l_pAttrRP = &TARG_GET_SINGLETON(TARGETING::theAttrRP);
+    ATTRIBUTE_ID* l_pAttrIds = nullptr;
+    AbstractPointer<void>* l_ppAttrAddrs = nullptr;
+    uint32_t l_numberMutexAttrsReset = 0;
+    uint32_t l_attrCount = 0;
+    l_attrCount = targetService().getTargetAttributes(const_cast<TARGETING::Target*>(i_pTarget),
+                                                      l_pAttrRP,
+                                                      l_pAttrIds,
+                                                      l_ppAttrAddrs );
+
+    for ( uint32_t l_attrIndex = 0; l_attrIndex < l_attrCount; l_attrIndex++)
+    {
+        const ATTRIBUTE_ID l_attrId = l_pAttrIds[l_attrIndex];
+        for( const auto mutexId : hbMutexAttrIds)
         {
-            const ATTRIBUTE_ID l_attrId = l_pAttrIds[l_attrIndex];
-            for( const auto mutexId : hbMutexAttrIds)
+            if(l_attrId == mutexId)
             {
-                if(l_attrId == mutexId)
+                mutex_t* l_mutex;
+                if(i_pTarget->_tryGetHbMutexAttr(l_attrId, l_mutex))
                 {
-                    mutex_t* l_mutex;
-                    if(l_pTarget->_tryGetHbMutexAttr(l_attrId, l_mutex))
-                    {
-                        mutex_init(l_mutex);
-                    }
-                    else
-                    {
-                        /*@
-                        *   @errortype         ERRORLOG::ERRL_SEV_PREDICTIVE
-                        *   @moduleid          TARG_SVC_RESET_MUTEX
-                        *   @reasoncode        TARG_SVC_MISSING_ATTR
-                        *   @userdata1         Attribute Id we attempted to read
-                        *   @userdata2         Huid of target we attempted to read
-                        *
-                        *   @devdesc   For some reason attr IDs in hbMutexAttrIds list
-                        *              are not matching the attribute IDs that target
-                        *              service is seeing. This is causing incorrect matching
-                        *              Make sure mutexattribute.H in genfiles has good values
-                        *
-                        *   @custdesc  Attempted to perform an invalid attribute look up
-                        */
-                        errlHndl_t l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_PREDICTIVE,
-                                                          TARG_SVC_RESET_MUTEX,
-                                                          TARG_SVC_MISSING_ATTR,
-                                                          l_attrId,
-                                                          get_huid(l_pTarget),
-                                                          true); // software error
-                        errlCommit(l_errl, TARG_COMP_ID);
-                    }
+                    mutex_init(l_mutex);
+                    l_numberMutexAttrsReset++;
+                }
+                else
+                {
+                    /*@
+                    *   @errortype         ERRORLOG::ERRL_SEV_PREDICTIVE
+                    *   @moduleid          TARG_SVC_RESET_MUTEX
+                    *   @reasoncode        TARG_SVC_MISSING_ATTR
+                    *   @userdata1         Attribute Id we attempted to read
+                    *   @userdata2         Huid of target we attempted to read
+                    *
+                    *   @devdesc   For some reason attr IDs in hbMutexAttrIds list
+                    *              are not matching the attribute IDs that target
+                    *              service is seeing. This is causing incorrect matching
+                    *              Make sure mutexattribute.H in genfiles has good values
+                    *
+                    *   @custdesc  Attempted to perform an invalid attribute look up
+                    */
+                    errlHndl_t l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_PREDICTIVE,
+                                                        TARG_SVC_RESET_MUTEX,
+                                                        TARG_SVC_MISSING_ATTR,
+                                                        l_attrId,
+                                                        get_huid(i_pTarget),
+                                                        true); // software error
+                    errlCommit(l_errl, TARG_COMP_ID);
                 }
             }
         }
     }
     #undef TARG_FN
+    return l_numberMutexAttrsReset;
 }
 #endif
 
diff --git a/src/usr/targeting/runtime/start_rt.C b/src/usr/targeting/runtime/start_rt.C
index ea99db972..d3edf9cb3 100644
--- a/src/usr/targeting/runtime/start_rt.C
+++ b/src/usr/targeting/runtime/start_rt.C
@@ -33,7 +33,7 @@ using namespace TARGETING;
 
 namespace RT_TARG
 {
-    void adjustTargeting4Runtime();
+    void adjustTargetingForRuntime();
 
     static void initTargeting() __attribute__((constructor));
     static void initTargeting()
@@ -50,17 +50,14 @@ namespace RT_TARG
         TargetService& l_targetService = targetService();
         l_targetService.init(Singleton<AttrRP>::instance().getNodeCount());
 
-        // Reset hb mutex attributes in case they got stuck in a locked state
-        l_targetService.resetMutexAttributes();
-
-        adjustTargeting4Runtime();
+        adjustTargetingForRuntime();
 
         // set global that TARG is ready
         Util::setIsTargetingLoaded();
     }
 
     // Make any adjustments needed to targeting for runtime
-    void adjustTargeting4Runtime()
+    void adjustTargetingForRuntime()
     {
         TRACDCOMP(g_trac_targeting,"adjustTargeting4Runtime");
 
@@ -77,109 +74,41 @@ namespace RT_TARG
         // was at IPL. Using this "old" address would seg fault at run time.
         // The value of the ATTR_PEER_TARGET attributes must be translated
         // for run time.
-        // The _trySetAttr is used directly to avoid the trySetAttr template
-        // error check that ATTR_PEER_TARGET is only readable, not writable.
-        // adjustTargeting4Runtime has been included as a friend to allow
-        // access to the private target class methods.
-        size_t l_xlateCnt = 0;
-        uint8_t l_maxNodeId =
-            TARGETING::targetService().getNumInitializedNodes();
+
+        // Also in this function we will reset any mutex attributes we find
+        // on any target incase they got left in the locked state when hostboot
+        // passed the payload to the hypervisor.
+
+        TargetService& l_targetService = targetService();
+        size_t   l_updatedCount = 0;
+        uint32_t l_numberMutexAttrsReset = 0;
+        uint8_t  l_maxNodeId = l_targetService.getNumInitializedNodes();
         for(uint8_t l_nodeId = NODE0; l_nodeId < l_maxNodeId; ++l_nodeId)
         {
-            for (TargetIterator target = targetService().begin(l_nodeId);
-                    target != targetService().end();
-                    ++target)
+            for (TargetIterator target = l_targetService.begin(l_nodeId);
+                 target != l_targetService.end();
+                 ++target)
             {
                 const TARGETING::Target * l_target = *target;
-                TARGETING::Target * l_peer =  static_cast<Target*>(NULL);
-                bool l_hasPeer = l_target->tryGetAttr<ATTR_PEER_TARGET>(l_peer);
-                if (l_hasPeer && (l_peer != nullptr))
-                {
-                    TRACDCOMP(g_trac_targeting,
-                            "translate the peer target %p for HUID %x",
-                            l_peer, get_huid(l_target));
+                // Check if there any mutex attributes we need to reset on this target
+                l_numberMutexAttrsReset += l_targetService.resetMutexAttributes(l_target);
 
-                    ATTR_PEER_TARGET_type l_xlated = (TARGETING::Target *)
-                                       Singleton<AttrRP>::instance().
-                                       AttrRP::translateAddr(l_peer,l_target);
-                    bool l_fixed = false;
-                    l_fixed = l_target->_trySetAttr(ATTR_PEER_TARGET,
-                                          sizeof(l_xlated),
-                                          &l_xlated);
-                    if (l_fixed)
-                    {
-                        TRACDCOMP(g_trac_targeting, "   to=%p", l_xlated);
-                        l_xlateCnt++;
-                    }
-                    // Not good if could not be fixed. But might not be
-                    // referenced. A segment fault will occur if used.
-                    else
-                    {
-                        TRACFCOMP(g_trac_targeting,
-                            "failed to translate peer target for HUID=0x%x",
-                            get_huid(l_target));
-                    }
-                }
-                else if(l_hasPeer && l_peer == nullptr)
+                // Check if there is any PEER_TARAGET attribute to update on this target
+                if(l_targetService.updatePeerTarget(l_target))
                 {
-                    TRACDCOMP(g_trac_targeting,
-                              "looking up peer path and target for HUID %x",
-                              get_huid(l_target));
-                    // Create variables entity path variable to write PEER_PATH into
-                    // as well as a Target pointer to set once we get the PEER_PATH
-                    TARGETING::EntityPath l_peerPath;
-                    TARGETING::Target * l_newTargPtr;
-
-                    // Look up the PEER_PATH attribute if it exists on the target
-                    bool l_hasPeerPath = l_target->tryGetAttr<ATTR_PEER_PATH>(l_peerPath);
-
-                    //If we find a PEER_PATH we need to next look up the PEER_TARGET
-                    if(l_hasPeerPath)
-                    {
-                        TRACDCOMP(g_trac_targeting,
-                                  "Found peer path for HUID %x",get_huid(l_target));
-                        // Look up the PEER_TARGET based on what the PEER_PATH is
-                        l_newTargPtr = targetService().toTarget(l_peerPath);
-
-                        bool l_fixed = false;
-
-                        // If the pointer returned from toTarget isn't null then
-                        // we will try to set PEER_TARGET with that value
-                        if(l_newTargPtr != nullptr)
-                        {
-                            l_fixed = l_target->_trySetAttr(ATTR_PEER_TARGET,
-                                                            sizeof(l_newTargPtr),
-                                                            &l_newTargPtr);
-                        }
-
-                        if (l_fixed)
-                        {
-                            TRACDCOMP(g_trac_targeting, "Peer target for HUID %x found to be %p",
-                                    get_huid(l_target), l_newTargPtr);
-                            l_xlateCnt++;
-                        }
-                        // Not good if could not be fixed. But might not be
-                        // referenced. A segment fault will occur if used.
-                        else
-                        {
-                            TRACFCOMP(g_trac_targeting,
-                                "failed to find peer target for HUID=0x%x",
-                                get_huid(l_target));
-                        }
-                    }
-                    else
-                    {
-                        TRACFCOMP(g_trac_targeting,
-                                  "Failed to find peer path for HUID=0x%x",
-                                  get_huid(l_target));
-                    }
+                    l_updatedCount++;
                 }
             }
         }
         TRACFCOMP(g_trac_targeting,
-                  "adjustTargeting4Runtime: %d peer target addresses "
+                  "adjustTargetingForRuntime: %d peer target addresses "
                   "translated on %d nodes",
-                  l_xlateCnt,
+                  l_updatedCount,
+                  l_maxNodeId);
+        TRACFCOMP(g_trac_targeting,
+                  "adjustTargetingForRuntime: %d mutex attributes reset "
+                  "on %d nodes",
+                  l_numberMutexAttrsReset,
                   l_maxNodeId);
     }
 }
diff --git a/src/usr/targeting/targetservicestart.C b/src/usr/targeting/targetservicestart.C
index 00393b017..5b7e639dd 100755
--- a/src/usr/targeting/targetservicestart.C
+++ b/src/usr/targeting/targetservicestart.C
@@ -194,7 +194,33 @@ static void initTargeting(errlHndl_t& io_pError)
         //Ensure all mutex attributes are reset on MPIPL
         if(l_isMpipl)
         {
-            l_targetService.resetMutexAttributes();
+            // updatePeerTargets will write to read-only attribute pages
+            // to get around vmm fails we need to allow writes to readonly
+            // memory for the duration of this loop
+            l_targetService.modifyReadOnlyPagePermissions(true);
+            uint32_t l_peerTargetsAdjusted = 0;
+            uint32_t l_numberMutexAttrsReset = 0;
+            for( auto targ_iter = l_targetService.begin();
+                 targ_iter != l_targetService.end();
+                 targ_iter++)
+            {
+                const Target* l_pTarget = *targ_iter;
+                // Check if there any mutex attributes we need to reset on this target
+                l_numberMutexAttrsReset += l_targetService.resetMutexAttributes(l_pTarget);
+
+                // Update any peer target addresses if necessary
+                if(l_targetService.updatePeerTarget(l_pTarget))
+                {
+                    l_peerTargetsAdjusted++;
+                }
+            }
+            // Now that the loop is complete we can re-apply
+            // the read only permissions to the read only attr pages
+            l_targetService.modifyReadOnlyPagePermissions(false);
+            TARG_INF("Number of peer target addresses adjusted: %d",
+                     l_peerTargetsAdjusted);
+            TARG_INF("Number of mutex attributes reset: %d",
+                     l_numberMutexAttrsReset);
         }
 
         checkProcessorTargeting(l_targetService);