Updating the Start of the SBEs on Slave Processors

This commit updates how the SBEs of Slave Processors are started,
including having them use the same SBE Image (either first or
second) that the Master Processor was started from.

Change-Id: I434ba387cdfd6b1f9695e8280b04734697f66222
RTC: 117705
CQ:SW317286
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/19570
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: MATTHEW A. PLOETZ <maploetz@us.ibm.com>
Reviewed-by: Joseph J. McGill <jmcgill@us.ibm.com>
Reviewed-by: Michael Baiocchi <baiocchi@us.ibm.com>
Tested-by: Michael Baiocchi <baiocchi@us.ibm.com>
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/17614
Tested-by: Jenkins Server
Tested-by: Jenkins OP Build CI
Tested-by: Jenkins OP HW
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

5 files changed, 255 insertions, 103 deletions

diff --git a/src/usr/hwpf/hwp/include/p8_scom_addresses.H b/src/usr/hwpf/hwp/include/p8_scom_addresses.H
index ba6ef0d27..a01834f97 100755
--- a/src/usr/hwpf/hwp/include/p8_scom_addresses.H
+++ b/src/usr/hwpf/hwp/include/p8_scom_addresses.H
@@ -22,7 +22,7 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
-// $Id: p8_scom_addresses.H,v 1.197 2015/06/29 01:43:56 jmcgill Exp $
+// $Id: p8_scom_addresses.H,v 1.198 2015/08/04 22:06:23 baiocchi Exp $
 // $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/p8_scom_addresses.H,v $
 //------------------------------------------------------------------------------
 // *! (C) Copyright International Business Machines Corp. 2011
@@ -160,6 +160,9 @@ CONST_UINT64_T( DEVICE_ID_REG_0x000F000F            , ULL(0x000F000F) );
 //------------------------------------------------------------------------------
 //      FSI MBOX (CFAM)
 //------------------------------------------------------------------------------
+CONST_UINT32_T( CFAM_FSI_INTR_MASK_0x0000081C         , ULL(0x0000081C) );
+CONST_UINT32_T( CFAM_FSI_INTR_STATUS_0x0000100B       , ULL(0x0000100B) );
+CONST_UINT32_T( CFAM_FSI_TRUE_MASK_0x0000100D         , ULL(0x0000100D) );
 CONST_UINT32_T( CFAM_FSI_GP3_0x00002812               , ULL(0x00002812) );
 CONST_UINT32_T( CFAM_FSI_GP4_0x00002813               , ULL(0x00002813) );
 CONST_UINT32_T( CFAM_FSI_GP5_0x00002814               , ULL(0x00002814) );
@@ -2262,6 +2265,9 @@ This section is automatically updated by CVS when you check in this file.
 Be sure to create CVS comments when you commit so that they can be included here.
 
 $Log: p8_scom_addresses.H,v $
+Revision 1.198  2015/08/04 22:06:23  baiocchi
+Updating Support for starting the SBEs without a FSP
+
 Revision 1.197  2015/06/29 01:43:56  jmcgill
 add definition for PCIE NEST FIR Action2 Registers (Naples only)
 
diff --git a/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.C b/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.C
index eb25ddba0..9ead7810e 100644
--- a/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.C
+++ b/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2014                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -22,10 +22,8 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
-// -*- mode: C++; c-file-style: "linux";  -*-
-
-// $Id: proc_spless_sbe_startWA.C,v 1.2 2013/08/14 20:44:47 jmcgill Exp $
-// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ipl/fapi/proc_cen_ref_clk_enable.C,v $
+// $Id: proc_spless_sbe_startWA.C,v 1.2 2015/08/05 14:19:34 baiocchi Exp $
+// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ipl/fapi/proc_spless_sbe_startWA.C,v $
 //------------------------------------------------------------------------------
 // *|
 // *! (C) Copyright International Business Machines Corp. 2013
@@ -60,9 +58,11 @@ extern "C"
 // Hardware Procedure
 //------------------------------------------------------------------------------
 // parameters: i_target            => chip target (S1/P8)
+//             i_sbeSeepromSelect  => SBE Seeprom Select Bit
 // returns:    FAPI_RC_SUCCESS if operation was successful, else error
 //------------------------------------------------------------------------------
-fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target)
+fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target,
+                                         const bool  i_sbeSeepromSelect)
 {
 
     uint32_t           rc_ecmd = 0;
@@ -70,6 +70,9 @@ fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target)
     uint32_t           l_set_data;
     ecmdDataBufferBase  set_data(32);
 
+    // mark start of function
+    FAPI_INF("proc_spless_sbe_startWA: Enter: i_sbeSeepromSelect=%d", i_sbeSeepromSelect);
+
     do {
 
         // -----------------------------------------------------------
@@ -99,12 +102,67 @@ fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target)
             break;
         }
 
+
         // ------------------------------------------------
-        // Now toggle Warmstart bit to circumvent HW254584
-        // write it to mbox scratch2
-        rc_ecmd |= set_data.setWord( 0, 0x30000000 );
+        // Enable output to ATTN pin to monitor for checkstops (3 commands)
+        // - putcfam pu 0x081C 20000000
+        FAPI_INF(   "Enable output to ATTN pin to monitor for checkstops");
+
+        rc_ecmd |= set_data.setWord( 0, 0x20000000 );
+
+        rc = fapiPutCfamRegister( i_target,
+                                  CFAM_FSI_INTR_MASK_0x0000081C,
+                                  set_data    );
+        if (rc)
+        {
+            FAPI_ERR("ERROR: write CFAM_FSI_INTR_MASK_0x0000081C= 0x%08x",
+                     static_cast<uint32_t>(rc) );
+            break;
+        }
+
+        // - putcfam pu 0x100D 40000000
+        rc_ecmd |= set_data.setWord( 0, 0x40000000 );
+
+        rc = fapiPutCfamRegister( i_target,
+                                  CFAM_FSI_TRUE_MASK_0x0000100D,
+                                  set_data    );
+        if (rc)
+        {
+            FAPI_ERR("ERROR: write CFAM_FSI_TRUE_MASK_0x0000100D= 0x%08x",
+                     static_cast<uint32_t>(rc) );
+            break;
+        }
 
 
+        // - putcfam pu  0x100B FFFFFFFF
+        rc_ecmd |= set_data.setWord( 0, 0xFFFFFFFF );
+
+        rc = fapiPutCfamRegister( i_target,
+                                  CFAM_FSI_INTR_STATUS_0x0000100B,
+                                  set_data    );
+        if (rc)
+        {
+            FAPI_ERR("ERROR: write CFAM_FSI_INTR_STATUS_0x0000100B= 0x%08x",
+                     static_cast<uint32_t>(rc) );
+            break;
+        }
+
+
+
+        // -----------------------------------------------
+        // Now toggle Warmstart bit to circumvent HW254584
+        // -- set bit 8 to select SBE Image to boot from
+        if (i_sbeSeepromSelect == true)
+        {
+            rc_ecmd |= set_data.setWord( 0, 0x30800000 );
+        }
+        else
+        {
+            rc_ecmd |= set_data.setWord( 0, 0x30000000 );
+        }
+        FAPI_INF(   "Write 0x%08x to SBE VITAL to toggle Warmstart bit",
+                    set_data.getWord(0)   );
+
         rc = fapiPutCfamRegister( i_target,
                                   CFAM_FSI_SBE_VITAL_0x0000281C,
                                   set_data    );
@@ -115,8 +173,20 @@ fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target)
             break;
         }
 
-        rc_ecmd |= set_data.setWord( 0, 0xB0000000 );
+        // -----------------------------------------------
+        // Now start SBE
+        // -- set bit 8 to select SBE Image to boot from
+        if (i_sbeSeepromSelect == true)
+        {
+            rc_ecmd |= set_data.setWord( 0, 0xB0800000 );
+        }
+        else
+        {
+            rc_ecmd |= set_data.setWord( 0, 0xB0000000 );
+        }
 
+        FAPI_INF(   "Write 0x%08x to SBE VITAL start SBE",
+                    set_data.getWord(0)   );
 
         rc = fapiPutCfamRegister( i_target,
                                   CFAM_FSI_SBE_VITAL_0x0000281C,
diff --git a/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.H b/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.H
index 4a3f474bc..29552ff54 100644
--- a/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.H
+++ b/src/usr/hwpf/hwp/slave_sbe/proc_spless_sbe_startWA/proc_spless_sbe_startWA.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2014                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -22,8 +22,8 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
-// $Id: proc_spless_sbe_startWA.H,v 1.1 2013/08/12 18:19:30 jmcgill Exp $
-// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ipl/fapi/proc_cen_ref_clk_enable.H,v $
+// $Id: proc_spless_sbe_startWA.H,v 1.1 2015/08/04 22:04:52 baiocchi Exp $
+// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ipl/fapi/proc_spless_sbe_startWA.H,v $
 //------------------------------------------------------------------------------
 // *|
 // *! (C) Copyright International Business Machines Corp. 2013
@@ -52,7 +52,8 @@
 //------------------------------------------------------------------------------
 
 // function pointer typedef definition for HWP call support
-typedef fapi::ReturnCode (*proc_spless_sbe_startWA_FP_t)(const fapi::Target &);
+typedef fapi::ReturnCode (*proc_spless_sbe_startWA_FP_t)(const fapi::Target &,
+                                                         const bool );
 
 
 //------------------------------------------------------------------------------
@@ -75,10 +76,13 @@ extern "C"
  *        started by Hostboot
  *
  * @param[in] i_target            chip target
+ * @param[in] i_sbeSeepromSelect  SBE Seeprom Select Bit:
+ *                                if false then first image; otherwise 2nd image
  *
  * @return ReturnCode
  */
-  fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target);
+  fapi::ReturnCode proc_spless_sbe_startWA(const fapi::Target & i_target,
+                                           const bool  i_sbeSeepromSelect);
 
 } // extern "C"
 
diff --git a/src/usr/hwpf/hwp/slave_sbe/slave_sbe.C b/src/usr/hwpf/hwp/slave_sbe/slave_sbe.C
index fe568e2e7..42a276cf8 100644
--- a/src/usr/hwpf/hwp/slave_sbe/slave_sbe.C
+++ b/src/usr/hwpf/hwp/slave_sbe/slave_sbe.C
@@ -268,85 +268,153 @@ void* call_host_sbe_start( void *io_pArgs )
 {
     errlHndl_t  l_errl = NULL;
     IStepError  l_stepError;
+    bool        l_sbeSeepromSelect = false;
 
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_host_sbe_start entry" );
 
-    //
-    //  get the master Proc target, we want to IGNORE this one.
-    //
-    TARGETING::Target* l_pMasterProcTarget = NULL;
-    TARGETING::targetService().masterProcChipTargetHandle(l_pMasterProcTarget);
-
-    //
-    //  get a list of all the procs in the system
-    //
-    TARGETING::TargetHandleList l_procTargetList;
-    getAllChips(l_procTargetList, TYPE_PROC);
+    do
+    {
+        //
+        //  get the master Proc target, we want to IGNORE this one.
+        //
+        TARGETING::Target* l_pMasterProcTarget = NULL;
+        TARGETING::targetService().masterProcChipTargetHandle(
+                                                 l_pMasterProcTarget);
+
+        //
+        //  get a list of all the procs in the system
+        //
+        TARGETING::TargetHandleList l_procTargetList;
+        getAllChips(l_procTargetList, TYPE_PROC);
+
+        TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+            "host_sbe_start: %d procs in the system.",
+            l_procTargetList.size() );
+
+
+        //
+        //  get the SBE Image side that master Proc booted from
+        //  if hostboot needs to start SBE on slave processors
+        //
+        if ( (l_procTargetList.size() > 0) &&
+             !INITSERVICE::spBaseServicesEnabled() )
+        {
+            uint64_t scomData = 0x0;
+            size_t op_size = sizeof(scomData);
+            l_errl = deviceRead(l_pMasterProcTarget,
+                                &scomData,
+                                op_size,
+                                DEVICE_SCOM_ADDRESS(PORE_SBE_VITAL_0x0005001C));
+
+            if(l_errl)
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           ERR_MRK"call_host_sbe_start: Error reading SBE "
+                           "VITAL REG (0x%.8X) from Target HUID=0x%.8X",
+                           PORE_SBE_VITAL_0x0005001C,
+                           TARGETING::get_huid(l_pMasterProcTarget));
 
-    TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
-        "host_sbe_start: %d procs in the system.",
-        l_procTargetList.size() );
+                // capture the target data in the elog
+                ErrlUserDetailsTarget(l_pMasterProcTarget).addToLog( l_errl );
 
-    // loop thru all the cpu's
-    for (TargetHandleList::const_iterator
-            l_proc_iter = l_procTargetList.begin();
-            l_proc_iter != l_procTargetList.end();
-            ++l_proc_iter)
-    {
-        //  make a local copy of the Processor target
-        TARGETING::Target* l_pProcTarget = *l_proc_iter;
+                l_errl->setSev(ERRL_SEV_INFORMATIONAL);
 
-        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "target HUID %.8X",
-                TARGETING::get_huid(l_pProcTarget));
+                // Create IStep error log and cross reference error
+                // that occurred
+                l_stepError.addErrorDetails( l_errl );
 
-        fapi::Target l_fapiProcTarget( fapi::TARGET_TYPE_PROC_CHIP,
-                                       l_pProcTarget    );
+                // Commit error log
+                errlCommit( l_errl, HWPF_COMP_ID );
 
+                // Don't break - attempt to continue
+            }
 
-        if (l_pProcTarget  ==  l_pMasterProcTarget )
-        {
-            // we are just checking the Slave SBE's, skip the master
-            continue;
-        }
-        else if (!INITSERVICE::spBaseServicesEnabled())
-        {
-            //Need to issue SBE start workaround on all slave chips
-            // Invoke the HWP
-            FAPI_INVOKE_HWP(l_errl,
-                            proc_spless_sbe_startWA,
-                            l_fapiProcTarget);
-        }
-        else
-        {
-            //Eventually for secureboot will need to kick off
-            //SBE here (different HWP), but for now not needed
-            //so do nothing
+            // Bit 8 in VITAL REG is SBE_IMAGE_SELECT bit:
+            // - if 0 then first image otherwise 2nd image
+            else if( scomData & 0x0080000000000000 )
+            {
+                l_sbeSeepromSelect = 1;
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           "call_host_sbe_start: Master Proc booted from "
+                           "2nd SBE Image. SBE_VITAL_REG(0x%X) data = 0x%.16X",
+                           PORE_SBE_VITAL_0x0005001C, scomData);
+            }
         }
 
-        if (l_errl)
+        // loop thru all the cpu's
+        for (TargetHandleList::const_iterator
+                l_proc_iter = l_procTargetList.begin();
+                l_proc_iter != l_procTargetList.end();
+                ++l_proc_iter)
         {
-            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                      "ERROR : sbe_start",
-                      "failed, returning errorlog" );
+            //  make a local copy of the Processor target
+            TARGETING::Target* l_pProcTarget = *l_proc_iter;
 
-            // capture the target data in the elog
-            ErrlUserDetailsTarget(l_pProcTarget).addToLog( l_errl );
+            fapi::Target l_fapiProcTarget( fapi::TARGET_TYPE_PROC_CHIP,
+                                           l_pProcTarget    );
 
-            // Create IStep error log and cross reference error that occurred
-            l_stepError.addErrorDetails( l_errl );
 
-            // Commit error log
-            errlCommit( l_errl, HWPF_COMP_ID );
-        }
-        else
-        {
+            if (l_pProcTarget  ==  l_pMasterProcTarget )
+            {
+                // we are just checking the Slave SBE's, skip the master
+               TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                        "target HUID %.8X --> skipping proc_spless_sbe_startWA",
+                        TARGETING::get_huid(l_pProcTarget));
+
+
+                continue;
+            }
+            else if (!INITSERVICE::spBaseServicesEnabled())
+            {
+                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                          "target HUID %.8X --> "
+                          "calling proc_spless_sbe_startWA",
+                          TARGETING::get_huid(l_pProcTarget));
+
+                //Need to issue SBE start workaround on all slave chips
+                // Invoke the HWP
+                FAPI_INVOKE_HWP(l_errl,
+                                proc_spless_sbe_startWA,
+                                l_fapiProcTarget,
+                                l_sbeSeepromSelect);
+            }
+            else
+            {
+                //Eventually for secureboot will need to kick off
+                //SBE here (different HWP), but for now not needed
+                //so do nothing
+            }
+
+            if (l_errl)
+            {
+                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                          "ERROR : sbe_start (proc_spless_sbe_startWA) "
+                          "failed, returning errorlog" );
+
+                // capture the target data in the elog
+                ErrlUserDetailsTarget(l_pProcTarget).addToLog( l_errl );
+
+                // Create IStep error log and cross reference error
+                // that occurred
+                l_stepError.addErrorDetails( l_errl );
+
+                // Commit error log
+                errlCommit( l_errl, HWPF_COMP_ID );
+            }
+            else
+            {
+                TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                          "SUCCESS : sbe_start (proc_spless_sbe_startWA) "
+                          "completed ok");
+
+            }
             TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                      "SUCCESS : sbe_start",
-                      "completed ok");
+                      "target HUID %.8X --> after proc_spless_sbe_startWA",
+                      TARGETING::get_huid(l_pProcTarget));
 
-        }
-    }   // endfor
+        }   // endfor
+
+    }while( 0 );
 
     TRACDCOMP( ISTEPS_TRACE::g_trac_isteps_trace, "call_host_sbe_start exit" );
 
@@ -498,7 +566,7 @@ void* call_proc_check_slave_sbe_seeprom_complete( void *io_pArgs )
         TARGETING::Target* l_pProcTarget = *l_proc_iter;
 
         TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
-                "target HUID %.8X",
+                "target HUID %.8X --> calling proc_getecid",
                 TARGETING::get_huid(l_pProcTarget));
 
         fapi::Target l_fapiProcTarget( fapi::TARGET_TYPE_PROC_CHIP,
@@ -535,8 +603,11 @@ void* call_proc_check_slave_sbe_seeprom_complete( void *io_pArgs )
                       " completed ok");
 
         }
-    }   // endfor
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                  "target HUID %.8X --> after proc_getecid",
+                  TARGETING::get_huid(l_pProcTarget));
 
+    }   // endfor
 
     // Slave processors should now use Host I2C Access Method
     I2C::i2cSetAccessMode( I2C::I2C_SET_ACCESS_MODE_PROC_HOST );
diff --git a/src/usr/sbe/sbe_update.C b/src/usr/sbe/sbe_update.C
index 4046e0c0c..9ab24b314 100644
--- a/src/usr/sbe/sbe_update.C
+++ b/src/usr/sbe/sbe_update.C
@@ -3047,66 +3047,67 @@ namespace SBE
 
         do{
 
-            // Set permissions back to "no_access" before returning
-            rc = mm_set_permission(reinterpret_cast<void*>
-                                   (VMM_VADDR_SBE_UPDATE),
-                                   VMM_SBE_UPDATE_SIZE,
-                                   NO_ACCESS | ALLOCATE_FROM_ZERO );
+            //release all pages in page block
+            rc = mm_remove_pages(RELEASE,
+                                 reinterpret_cast<void*>
+                                 (VMM_VADDR_SBE_UPDATE),
+                                 VMM_SBE_UPDATE_SIZE);
             if( rc )
             {
                 TRACFCOMP( g_trac_sbe, ERR_MRK"cleanupSbeImageVmmSpace() - "
-                           "Error from mm_set_permission : rc=%d", rc );
+                           "Error from mm_remove_pages : rc=%d", rc );
                 /*@
                  * @errortype
                  * @moduleid     SBE_CLEANUP_TEST_SPACE
-                 * @reasoncode   SBE_SET_PERMISSION_FAIL
+                 * @reasoncode   SBE_REMOVE_PAGES_FAIL
                  * @userdata1    Requested Address
-                 * @userdata2    rc from mm_set_permission
-                 * @devdesc      updateProcessorSbeSeeproms> Error from
-                 *               mm_set_permission on cleanup
+                 * @userdata2    rc from mm_remove_pages
+                 * @devdesc      updateProcessorSbeSeeproms> mm_remove_pages
+                 *               RELEASE failed
                  * @custdesc     A problem occurred while updating processor
                  *               boot code.
                  */
                 err = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
                                     SBE_CLEANUP_TEST_SPACE,
-                                    SBE_SET_PERMISSION_FAIL,
+                                    SBE_REMOVE_PAGES_FAIL,
                                     TO_UINT64(VMM_VADDR_SBE_UPDATE),
                                     TO_UINT64(rc));
                 err->collectTrace(SBE_COMP_NAME);
                 err->addProcedureCallout( HWAS::EPUB_PRC_HB_CODE,
                                           HWAS::SRCI_PRIORITY_HIGH );
+
                 break;
             }
 
-            //release all pages in page block to ensure we
-            //start with clean state
-            rc = mm_remove_pages(RELEASE,
-                                 reinterpret_cast<void*>
-                                 (VMM_VADDR_SBE_UPDATE),
-                                 VMM_SBE_UPDATE_SIZE);
+            // Set permissions back to "no_access"
+            rc = mm_set_permission(reinterpret_cast<void*>
+                                   (VMM_VADDR_SBE_UPDATE),
+                                   VMM_SBE_UPDATE_SIZE,
+                                   NO_ACCESS | ALLOCATE_FROM_ZERO );
             if( rc )
             {
                 TRACFCOMP( g_trac_sbe, ERR_MRK"cleanupSbeImageVmmSpace() - "
-                           "Error from mm_remove_pages : rc=%d", rc );
+                           "Error from mm_set_permission : rc=%d", rc );
                 /*@
                  * @errortype
                  * @moduleid     SBE_CLEANUP_TEST_SPACE
-                 * @reasoncode   SBE_REMOVE_PAGES_FAIL
+                 * @reasoncode   SBE_SET_PERMISSION_FAIL
                  * @userdata1    Requested Address
-                 * @userdata2    rc from mm_remove_pages
-                 * @devdesc      updateProcessorSbeSeeproms> mm_remove_pages
-                 *               RELEASE failed
+                 * @userdata2    rc from mm_set_permission
+                 * @devdesc      updateProcessorSbeSeeproms> Error from
+                 *               mm_set_permission on cleanup
                  * @custdesc     A problem occurred while updating processor
                  *               boot code.
                  */
                 err = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
                                     SBE_CLEANUP_TEST_SPACE,
-                                    SBE_REMOVE_PAGES_FAIL,
+                                    SBE_SET_PERMISSION_FAIL,
                                     TO_UINT64(VMM_VADDR_SBE_UPDATE),
                                     TO_UINT64(rc));
                 err->collectTrace(SBE_COMP_NAME);
                 err->addProcedureCallout( HWAS::EPUB_PRC_HB_CODE,
                                           HWAS::SRCI_PRIORITY_HIGH );
+                break;
             }
 
         }while(0);