Implement Secure unload

Implement Secure unload of secure sections within PNOR.

Change-Id: I92a00013d23e0506f89f89ec41a193eac0b25d25
RTC:157475
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/46203
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

9 files changed, 516 insertions, 77 deletions

diff --git a/src/include/usr/pnor/pnor_reasoncodes.H b/src/include/usr/pnor/pnor_reasoncodes.H
index 3eee83930..e9e98f9c5 100644
--- a/src/include/usr/pnor/pnor_reasoncodes.H
+++ b/src/include/usr/pnor/pnor_reasoncodes.H
@@ -105,9 +105,10 @@ namespace PNOR
         MOD_SPNORRP_WAITFORMESSAGE      = 0xD3, /**< SPnorRP::waitForMessage */
         MOD_SPNORRP_VERIFYSECTIONS      = 0xD4, /**< SPnorRP::verifySections */
         MOD_SPNORRP_SET_PERMISSION      = 0xD5, /**< SPnorRP::initDaemon */
-        MOD_PNORRP_LOADSECURESECTION    = 0xD6, /**< PnorRP::loadSecureSection */
+        MOD_PNORRP_LOADUNLOADSECURESECTION = 0xD6, /**< PnorRP::loadSecureSection */
         MOD_SPNORRP_BASE_EXT_VER_CHK    = 0xD7, /**< SPnorRP::baseExtVersCheck */
         MOD_SPNORRP_KEY_TRAN_CHK        = 0xD8, /**< SPnorRP::keyTransitionCheck */
+        MOD_SPNORRP_REMOVE_PAGES        = 0xD9,
 
         // ast_mboxdd.C
         MOD_ASTMBOXDD_DO_MESSAGE        = 0xE0, /**< astMbox::doMessage */
@@ -178,6 +179,9 @@ namespace PNOR
         RC_MBOX_BAD_SEQUENCE         = PNOR_COMP_ID | 0x32,
         RC_MBOX_ERROR_STATUS         = PNOR_COMP_ID | 0x33,
         RC_UNSIGNED_PNOR_SECTION     = PNOR_COMP_ID | 0x34,
+        RC_NOT_A_LOADED_SECTION      = PNOR_COMP_ID | 0x35,
+        RC_NOT_A_SUPPORTED_SECTION   = PNOR_COMP_ID | 0x36,
+        RC_SECURE_UNLOAD_DISALLOWED  = PNOR_COMP_ID | 0x37,
 
         //@fixme-RTC:131607-Temporary value to allow HWSV compile
         //termination_rc
diff --git a/src/include/usr/pnor/pnorif.H b/src/include/usr/pnor/pnorif.H
index 7e62e6042..1c35ec852 100644
--- a/src/include/usr/pnor/pnorif.H
+++ b/src/include/usr/pnor/pnorif.H
@@ -94,8 +94,9 @@ errlHndl_t getSectionInfo( SectionId i_section,
  *      placing it in the address space.  Returns error if the requested
  *      section does not have secure space support.
  *
- *  @param[in] i_section PNOR section to load. Section must not already be
- *      loaded.
+ *  @param[in] i_section PNOR section to load. If the section is already loaded,
+ *                       the bulk of the load operation will not be repeated as
+ *                       a reference count is maintained for each section.
  *
  *  @return errlHndl_t Error log handle
  *  @retval NULL Successfully loaded PNOR section
@@ -104,14 +105,20 @@ errlHndl_t getSectionInfo( SectionId i_section,
 errlHndl_t loadSecureSection(SectionId i_section);
 
 /**
- *  @brief Flushes any applicable pending writes and unloads requested PNOR
- *      section from secure virtual address space
+ *  @brief Unloads requested PNOR section from secure virtual address space.
+ *         There are some restrictions about which sections can be unloaded.
+ *         1. You cannot unload HBB, HBI, or targeting sections. These are core
+ *            sections that need to be available as much as possible.
+ *         2. You can only unload sections for which the secure payload is
+ *            the entire payload. Sections with unsecured pages are not
+ *            supported.
  *
- *  @param[in] i_section PNOR section to unload.  No-op if already unloaded.
+ *  @param[in] i_section PNOR section to unload.
  *
  *  @return errlHndl_t Error log handle
  *  @retval NULL Successfully unloaded PNOR section
- *  @retval !NULL Failed to unload PNOR section
+ *  @retval !NULL Failed to unload PNOR section for the reasons listed above
+ *                or if the section is not currently loaded.
  */
 errlHndl_t unloadSecureSection(SectionId i_section);
 
diff --git a/src/usr/pnor/pnorrp.C b/src/usr/pnor/pnorrp.C
index 5a79879e2..0d4915ed0 100644
--- a/src/usr/pnor/pnorrp.C
+++ b/src/usr/pnor/pnorrp.C
@@ -118,8 +118,17 @@ errlHndl_t PNOR::flush( PNOR::SectionId i_section)
                     " secId: %d", (int)i_section);
             break;
         }
-        int l_rc = mm_remove_pages (RELEASE,
-                reinterpret_cast<void*>(l_info.vaddr), l_info.size);
+        uint8_t* l_vaddr = reinterpret_cast<uint8_t*>(l_info.vaddr);
+        #ifdef CONFIG_SECUREBOOT
+        if (l_info.secure)
+        {
+            // subtract 2 deltas to get the PNOR unsecured address
+            l_vaddr = l_vaddr
+               - VMM_VADDR_SPNOR_DELTA
+               - VMM_VADDR_SPNOR_DELTA;
+        }
+        #endif
+        int l_rc = mm_remove_pages (RELEASE, l_vaddr, l_info.size);
         if (l_rc)
         {
             TRACFCOMP(g_trac_pnor, "PNOR::flush: mm_remove_pages errored,"
diff --git a/src/usr/pnor/spnorrp.C b/src/usr/pnor/spnorrp.C
index c4fae6937..c2a39e952 100644
--- a/src/usr/pnor/spnorrp.C
+++ b/src/usr/pnor/spnorrp.C
@@ -117,6 +117,8 @@ SPnorRP::~SPnorRP()
         ++i)
     {
         LoadRecord* l_rec = (*i).second;
+        TRACFCOMP(g_trac_pnor,
+            "Section 0x%X has %lu references.", (*i).first, l_rec->refCount);
         delete l_rec;
     }
 
@@ -195,6 +197,44 @@ errlHndl_t SPnorRP::setPermission(void* i_va, uint64_t i_size,
 }
 
 /**
+ * @brief  A wrapper for mm_remove_pages that adds error log creation.
+ */
+errlHndl_t SPnorRP::removePages(void* i_va, uint64_t i_size) const
+{
+    errlHndl_t l_errhdl = nullptr;
+    int l_rc = mm_remove_pages (RELEASE,
+            reinterpret_cast<void*>(i_va), i_size);
+    if (l_rc)
+    {
+        TRACFCOMP(g_trac_pnor, "SPnorRP::removePages: mm_remove_pages errored,"
+                " vaddr: 0x%llX, rc: %d, size:0x%llX", i_va, l_rc, i_size);
+        /*@
+         *  @errortype
+         *  @moduleid         PNOR::MOD_SPNORRP_REMOVE_PAGES
+         *  @reasoncode       PNOR::RC_EXTERNAL_ERROR
+         *  @userdata1        virtual address
+         *  @userdata2[00:31] rc from mm_remove_pages
+         *  @userdata2[32:63] The size of memory attempted to remove
+         *  @devdesc          mm_remove_pages failed
+         *  @custdesc         A problem occurred in the security subsystem
+         */
+        l_errhdl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                    PNOR::MOD_SPNORRP_REMOVE_PAGES,
+                                    PNOR::RC_EXTERNAL_ERROR,
+                                    reinterpret_cast<uint64_t>(i_va),
+                                    TWO_UINT32_TO_UINT64(
+                                        TO_UINT32(l_rc),
+                                        TO_UINT32(i_size)
+                                    ),
+                                    true); // Add HB SW Callout
+        l_errhdl->collectTrace(PNOR_COMP_NAME);
+        l_errhdl->collectTrace(SECURE_COMP_NAME);
+    }
+    return l_errhdl;
+}
+
+
+/**
  * @brief Initialize the daemon
  */
 void SPnorRP::initDaemon()
@@ -222,7 +262,7 @@ void SPnorRP::initDaemon()
 
         // set permissions for temp space
         l_errhdl = setPermission(reinterpret_cast<void*>(TEMP_VADDR),
-                                    PNOR_SIZE, WRITABLE | ALLOCATE_FROM_ZERO);
+                                                        PNOR_SIZE, NO_ACCESS);
         if ( l_errhdl )
         {
             break;
@@ -261,7 +301,9 @@ void SPnorRP::initDaemon()
 /**
  * @brief  Load secure sections into temporary address space and verify them
  */
-uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
+uint64_t SPnorRP::verifySections(SectionId i_id,
+                                 bool i_loadedPreviously,
+                                 LoadRecord* io_rec)
 {
     SectionInfo_t l_info;
     errlHndl_t l_errhdl = NULL;
@@ -348,6 +390,28 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
         TRACDBIN(g_trac_pnor,"SPnorRP::verifySections unsecured mem now: ",
                              l_unsecuredAddr, 128);
 
+        TRACDCOMP( g_trac_pnor,
+                    "SPnorRP::verifySections Doing setPermission for address "
+                              "0x%llX of length 0x%llX",
+                       l_tempAddr,
+                       l_info.secureProtectedPayloadSize + PAGESIZE);
+
+        l_errhdl = setPermission(l_tempAddr,
+                                 l_info.secureProtectedPayloadSize + PAGESIZE,
+                                 WRITABLE | ALLOCATE_FROM_ZERO);
+
+        if (l_errhdl)
+        {
+            TRACFCOMP( g_trac_pnor,
+                       ERR_MRK"SPnorRP::verifySections "
+                              "setPermission failed for address "
+                              "0x%llX of length 0x%llX",
+                       l_tempAddr,
+                       l_info.secureProtectedPayloadSize + PAGESIZE);
+            break;
+        }
+
+
         TRACDCOMP(g_trac_pnor,"SPnorRP::verifySections about to do memcpy");
 
         // copy from unsecured PNOR space to temp PNOR space
@@ -380,10 +444,10 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
                              l_tempAddr, 128);
 
         // store secure space pointer in load record (Includes Header)
-        o_rec->secAddr = reinterpret_cast<uint8_t*>(l_info.vaddr);
+        io_rec->secAddr = reinterpret_cast<uint8_t*>(l_info.vaddr);
 
         TRACDCOMP(g_trac_pnor,"section start address in secure space is "
-                              "0x%.16llX",o_rec->secAddr);
+                              "0x%.16llX",io_rec->secAddr);
 
         // verify while in temp space
         if (SECUREBOOT::enabled())
@@ -420,29 +484,45 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
         // parse container header now that it is verified
         // store the payload text size in the section load record
         // Note: the text size we get back is now trusted
-        o_rec->textSize = l_conHdr.payloadTextSize();
-        assert(o_rec->textSize == l_info.secureProtectedPayloadSize);
+        io_rec->textSize = l_conHdr.payloadTextSize();
+        assert(io_rec->textSize == l_info.secureProtectedPayloadSize);
         // Size of data loaded into Secure PnorRP vaddr space (Includes Header)
-        size_t l_protectedSizeWithHdr = PAGESIZE + o_rec->textSize;
+        size_t l_protectedSizeWithHdr = PAGESIZE + io_rec->textSize;
         TRACFCOMP(g_trac_pnor, "SPnorRP::verifySections Total Protected size with Header = 0x%.16llX",
                   l_protectedSizeWithHdr);
 
         l_totalContainerSize = l_conHdr.totalContainerSize();
         // keep track of info size in load record (Includes Header)
-        o_rec->infoSize = l_totalContainerSize;
-
-        // pcr extension of PNOR hash
-        l_errhdl = TRUSTEDBOOT::extendPnorSectionHash(l_conHdr,
+        io_rec->infoSize = l_totalContainerSize;
+
+        // if not loaded previously or...
+        if (!i_loadedPreviously ||
+        // loading after a prior unload and...
+            (i_loadedPreviously &&
+            // the protected payload measurement doesn't match the old one
+            memcmp(&io_rec->payloadTextHash[0],
+                   l_conHdr.payloadTextHash(),
+                   SHA512_DIGEST_LENGTH)!=0
+            )
+           )
+        {
+            // pcr extension of PNOR hash
+            l_errhdl = TRUSTEDBOOT::extendPnorSectionHash(l_conHdr,
                                         (l_tempAddr + PAGESIZE),
                                         i_id);
-        if(l_errhdl)
-        {
-            TRACFCOMP(g_trac_pnor,"SPnorRP::verifySections extendPnorSectionHash failed");
-            break;
+            if(l_errhdl)
+            {
+                TRACFCOMP(g_trac_pnor,"SPnorRP::verifySections extendPnorSectionHash failed");
+                break;
+            }
+            // save off the payload text hash
+            memcpy(&io_rec->payloadTextHash[0],
+                   l_conHdr.payloadTextHash(),
+                   SHA512_DIGEST_LENGTH);
         }
 
         // set permissions on the secured pages to writable
-        l_errhdl = setPermission(o_rec->secAddr, l_protectedSizeWithHdr,
+        l_errhdl = setPermission(io_rec->secAddr, l_protectedSizeWithHdr,
                                  WRITABLE);
         if(l_errhdl)
         {
@@ -454,7 +534,7 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
         // set permissions on the unsecured pages to write tracked so that any
         // unprotected payload pages with dirty writes can flow back to PNOR.
         uint64_t unprotectedPayloadSize = l_totalContainerSize
-            - PAGESIZE - o_rec->textSize;
+            - PAGESIZE - io_rec->textSize;
         if (unprotectedPayloadSize) // only write track a non-zero range
         {
             TRACDCOMP(g_trac_pnor,INFO_MRK " SPnorRP::verifySections "
@@ -464,13 +544,13 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
 
             // Split the mod math out of the assert as the trace would not
             // display otherwise.
-            bool l_onPageBoundary = !(o_rec->textSize % PAGESIZE);
+            bool l_onPageBoundary = !(io_rec->textSize % PAGESIZE);
             assert( l_onPageBoundary, "For section %s, payloadTextSize does "
                     "not fall on a page boundary and there is an unprotected "
                     "payload",
                     l_info.name);
 
-            l_errhdl = setPermission(o_rec->secAddr + l_protectedSizeWithHdr,
+            l_errhdl = setPermission(io_rec->secAddr + l_protectedSizeWithHdr,
                                        unprotectedPayloadSize,
                                        WRITABLE | WRITE_TRACKED);
             if(l_errhdl)
@@ -482,7 +562,7 @@ uint64_t SPnorRP::verifySections(SectionId i_id, LoadRecord* o_rec)
 
             // Register the write tracked memory range to be flushed on
             // shutdown.
-            INITSERVICE::registerBlock(o_rec->secAddr + l_protectedSizeWithHdr,
+            INITSERVICE::registerBlock(io_rec->secAddr + l_protectedSizeWithHdr,
                                         unprotectedPayloadSize, SPNOR_PRIORITY);
         }
         else
@@ -646,26 +726,246 @@ void SPnorRP::waitForMessage()
                                     static_cast<SectionId>(message->data[0]);
                         do
                         {
-                            if (iv_loadedSections[l_id] == NULL)
+                            LoadRecord* l_record = nullptr;
+                            bool l_loadedPreviously = false;
+                            uint64_t l_rc = 0;
+
+                            // if there is already a Load Record
+                            auto l_item = iv_loadedSections.find(l_id);
+                            if (l_item != iv_loadedSections.end())
                             {
-                                LoadRecord* l_record = new LoadRecord;
-                                uint64_t l_rc = verifySections(l_id, l_record);
+                                l_loadedPreviously = true;
+                                l_record = iv_loadedSections[l_id];
+                            }
+                            else
+                            {
+                                l_record = new LoadRecord;
+                            }
+
+                            TRACDCOMP(g_trac_pnor, "SPnorRP::waitForMessage> MSG_LOAD_SECTION refCount is %i",l_record->refCount);
+                            if (l_record->refCount == 0)
+                            {
+                                l_rc = verifySections(l_id,
+                                                      l_loadedPreviously,
+                                                      l_record);
                                 if (l_rc)
                                 {
-                                    delete l_record;
-                                    l_record = NULL;
+                                    if(!l_loadedPreviously)
+                                    {
+                                        delete l_record;
+                                        l_record = nullptr;
+                                    }
                                     status_rc = -l_rc;
                                     break;
                                 }
-                                iv_loadedSections[l_id] = l_record;
+                            }
 
-                                // cache the record to use fields later as hints
-                                l_rec = *l_record;
+                            if (!l_loadedPreviously)
+                            {
+                                iv_loadedSections[l_id] = l_record;
                             }
+
+                            // increment refcount
+                            l_record->refCount++;
+
+                            // cache the record to use fields later as hints
+                            l_rec = *l_record;
+
                         } while (0);
                     }
                     break;
+                case( PNOR::MSG_UNLOAD_SECTION ):
+                    {
+                        SectionId l_id =
+                                      static_cast<SectionId>(message->data[0]);
+
+                        do {
+                            // Disallow unload of HBB, HBI and Targeting
+                            if (l_id == HB_BASE_CODE ||
+                                l_id == HB_EXT_CODE ||
+                                l_id == HB_DATA)
+                            {
+                                TRACFCOMP( g_trac_pnor, ERR_MRK"SPnorRP::waitForMessage> Secure unload of HBB, HBI, and targeting is not allowed secId=%d", l_id);
+                                /*@
+                                 * @errortype
+                                 * @moduleid   PNOR::MOD_SPNORRP_WAITFORMESSAGE
+                                 * @reasoncode PNOR::RC_SECURE_UNLOAD_DISALLOWED
+                                 * @userdata1  Section Id
+                                 * @devdesc    Secure unload of sections that
+                                 *             critical to hostboot operation
+                                 *             are not allowed.
+                                 * @custdesc   A problem occurred within the
+                                 *             security subsystem.
+                                 */
+                                l_errhdl = new ERRORLOG::ErrlEntry(
+                                    ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                    PNOR::MOD_SPNORRP_WAITFORMESSAGE,
+                                    PNOR::RC_SECURE_UNLOAD_DISALLOWED,
+                                    TO_UINT64(l_id),
+                                    0,
+                                    true /*Add HB SW Callout*/);
+                                status_rc = -EFAULT;
+                                break;
+
+                            }
+
+                            // if we don't find a record
+                            // or refcount is zero then throw an error since
+                            // this is not currently a loaded section
+                            auto l_item = iv_loadedSections.find(l_id);
+                            if (l_item == iv_loadedSections.end() ||
+                                l_item->second->refCount == 0 )
+                            {
+                                TRACFCOMP( g_trac_pnor, ERR_MRK"SPnorRP::waitForMessage> Attempting to unload a section that is not a loaded section. refCount=%i",l_item->second->refCount);
+                                /*@
+                                 * @errortype
+                                 * @moduleid   PNOR::MOD_SPNORRP_WAITFORMESSAGE
+                                 * @reasoncode PNOR::RC_NOT_A_LOADED_SECTION
+                                 * @userdata1  Section attempted to unload
+                                 * @devdesc    Not a loaded section
+                                 * @custdesc   A problem occurred within the
+                                 *             security subsystem.
+                                 */
+                                l_errhdl = new ERRORLOG::ErrlEntry(
+                                    ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                    PNOR::MOD_SPNORRP_WAITFORMESSAGE,
+                                    PNOR::RC_NOT_A_LOADED_SECTION,
+                                    TO_UINT64(l_id),
+                                    0,
+                                    true /*Add HB SW Callout*/);
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            auto l_rec = l_item->second;
+
+                            TRACDCOMP(g_trac_pnor, "SPnorRP::waitForMessage> MSG_UNLOAD_SECTION refCount is %i",l_rec->refCount);
+
+                            size_t l_sizeWithHdr = PAGESIZE + l_rec->textSize;
 
+                            bool l_wasLoadedAsBestEffort = false;
+                            if (l_rec->textSize == 0 &&
+                                SECUREBOOT::bestEffortPolicy())
+                            {
+                                // indicate that this section had been loaded
+                                // as "best effort"
+                                l_wasLoadedAsBestEffort = true;
+                            }
+                            // if the section has an unsecured portion
+                            else if (l_sizeWithHdr != l_rec->infoSize)
+                            {
+                                TRACFCOMP( g_trac_pnor, ERR_MRK"SPnorRP::waitForMessage> Attempting to unload an unsupported section: 0x%X textsize+hdr: 0x%llX infosize: 0x%llX (the two sizes must be equal)", l_id, l_sizeWithHdr, l_rec->infoSize);
+                                /*@
+                                 * @errortype
+                                 * @moduleid   PNOR::MOD_SPNORRP_WAITFORMESSAGE
+                                 * @reasoncode PNOR::RC_NOT_A_SUPPORTED_SECTION
+                                 * @userdata1  Section attempted to unload
+                                 * @devdesc    Not a supported section
+                                 * @custdesc   A problem occurred within the
+                                 *             security subsystem.
+                                 */
+                                l_errhdl = new ERRORLOG::ErrlEntry(
+                                    ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                    PNOR::MOD_SPNORRP_WAITFORMESSAGE,
+                                    PNOR::RC_NOT_A_SUPPORTED_SECTION,
+                                    TO_UINT64(l_id),
+                                    0,
+                                    true /*Add HB SW Callout*/);
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            if (l_rec->refCount > 1)
+                            {
+                                l_rec->refCount--;
+                                 // normal operation, no error
+                                 // no need to do anything if refcount is
+                                 // 2 or more
+                                break;
+                            }
+
+                            if (l_wasLoadedAsBestEffort)
+                            {
+                                l_rec->secAddr = nullptr;
+                                l_rec->textSize = 0;
+                                l_rec->infoSize = 0;
+                                l_rec->refCount = 0;
+                                break;
+                            }
+
+                            l_errhdl = removePages(l_rec->secAddr,
+                                                   l_sizeWithHdr);
+                            if (l_errhdl)
+                            {
+                                TRACFCOMP( g_trac_pnor,
+                                    ERR_MRK"SPnorRP::waitForMessage> "
+                                    "removePages failed for address "
+                                    "0x%llX of length 0x%llX", l_rec->secAddr,
+                                                               l_sizeWithHdr);
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            l_errhdl = setPermission(l_rec->secAddr,
+                                                      l_sizeWithHdr,
+                                                      NO_ACCESS);
+                            if (l_errhdl)
+                            {
+                                TRACFCOMP( g_trac_pnor,
+                                    ERR_MRK"SPnorRP::waitForMessage> "
+                                    "setPermission failed for address "
+                                    "0x%llX of length 0x%llX", l_rec->secAddr,
+                                                           l_sizeWithHdr);
+
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            // clear out temp space
+                            uint8_t* l_tempAddr = l_rec->secAddr -
+                                                    VMM_VADDR_SPNOR_DELTA;
+
+                            l_errhdl = removePages(l_tempAddr,
+                                                   l_sizeWithHdr);
+                            if (l_errhdl)
+                            {
+                                TRACFCOMP( g_trac_pnor,
+                                    ERR_MRK"SPnorRP::waitForMessage> "
+                                    "removePages failed for address "
+                                    "0x%llX of length 0x%llX", l_tempAddr,
+                                                               l_sizeWithHdr);
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            TRACDCOMP( g_trac_pnor,
+                                    "SPnorRP::waitForMessage> "
+                                    "Doing setPermission NO_ACCESS for address "
+                                    "0x%llX of length 0x%llX", l_tempAddr,
+                                                               l_sizeWithHdr);
+
+                            l_errhdl = setPermission(l_tempAddr,
+                                                      l_sizeWithHdr,
+                                                      NO_ACCESS);
+                            if (l_errhdl)
+                            {
+                                TRACFCOMP( g_trac_pnor,
+                                    ERR_MRK"SPnorRP::waitForMessage> "
+                                    "setPermission failed for address "
+                                    "0x%llX of length 0x%llX", l_tempAddr,
+                                                               l_sizeWithHdr);
+
+                                status_rc = -EFAULT;
+                                break;
+                            }
+
+                            l_rec->secAddr = nullptr;
+                            l_rec->textSize = 0;
+                            l_rec->infoSize = 0;
+                            l_rec->refCount = 0;
+                        } while (0);
+                    }
+                    break;
                 default:
                     TRACFCOMP( g_trac_pnor, ERR_MRK"SPnorRP::waitForMessage> "
                     "Unrecognized message type : user_addr=%p, eff_addr=%p,"
@@ -699,6 +999,8 @@ void SPnorRP::waitForMessage()
             if( l_errhdl )
             {
                 errlCommit(l_errhdl,PNOR_COMP_ID);
+                TRACFCOMP(g_trac_pnor,
+                    ERR_MRK"SPnorRP::waitForMessage> status_rc=%d, ", status_rc );
             }
 
             /*  Expected Response:
@@ -725,10 +1027,15 @@ void SPnorRP::waitForMessage()
 }
 
 /**
- *  @brief Loads requested PNOR section to secure virtual address space
+ *  @brief Loads or unloads requested PNOR section to secure virtual address
+ *         space
  */
-errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
+errlHndl_t loadUnloadSecureSection(const SectionId i_section,
+                                   secure_msg_type i_loadUnload)
 {
+    assert( i_loadUnload == PNOR::MSG_LOAD_SECTION ||
+            i_loadUnload == PNOR::MSG_UNLOAD_SECTION, "Bug! You can only pass PNOR::MSG_LOAD_SECTION or PNOR::MSG_UNLOAD_SECTION into loadUnloadSecureSection()");
+
     // Send message to secure provider to load the section
     errlHndl_t err = NULL;
 
@@ -740,13 +1047,10 @@ errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
 
     assert(msg != NULL);
 
-    msg->type = PNOR::MSG_LOAD_SECTION;
+    msg->type = i_loadUnload;
     msg->data[0] = static_cast<uint64_t>(i_section);
     int rc = msg_sendrecv(spnorQ, msg);
 
-    TRACFCOMP(g_trac_pnor, "loadSecureSection i_section = %i (%s)",
-              i_section,PNOR::SectionIdToString(i_section));
-
     // TODO securebootp9 - Need to be able to receive an error from the
     // message handler. Also, message handler should police whether the request
     // is for a secure section or not and throw an error accordingly.
@@ -755,16 +1059,18 @@ errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
     //    err = reinterpret_cast<errlHndl_t>(msg->data[1]);
     //}
     //else remove the if clause below at some point
+    TRACDCOMP(g_trac_pnor,"PNOR::loadUnloadSecureSection> rc=%d msg->data[1]=0x%X ", rc, msg->data[1] );
+
     if (rc != 0 || msg->data[1])
     {
         // Use rc if non-zero, msg data[1] otherwise.
         uint64_t l_rc = (rc) ? rc : msg->data[1];
 
-        TRACFCOMP(g_trac_pnor,ERR_MRK"PNOR::loadSecureSection> Error from msg_sendrecv or msg->data[1] rc=0x%X",
+        TRACFCOMP(g_trac_pnor,ERR_MRK"PNOR::loadUnloadSecureSection> Error from msg_sendrecv or msg->data[1] rc=%d",
                   l_rc );
         /* @errorlog
          * @severity          ERRL_SEV_CRITICAL_SYS_TERM
-         * @moduleid          MOD_PNORRP_LOADSECURESECTION
+         * @moduleid          MOD_PNORRP_LOADUNLOADSECURESECTION
          * @reasoncode        RC_EXTERNAL_ERROR
          * @userdata1         returncode from msg_sendrecv() or msg->data[1]
          * @userdata2[0:31]   SPNOR message type [LOAD | UNLOAD]
@@ -775,10 +1081,10 @@ errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
          */
         err = new ERRORLOG::ErrlEntry(
                          ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
-                         MOD_PNORRP_LOADSECURESECTION,
+                         MOD_PNORRP_LOADUNLOADSECURESECTION,
                          RC_EXTERNAL_ERROR,
                          l_rc,
-                         TWO_UINT32_TO_UINT64(PNOR::MSG_LOAD_SECTION,
+                         TWO_UINT32_TO_UINT64(i_loadUnload,
                                               i_section),
                          true /* Add HB Software Callout */);
         err->collectTrace(PNOR_COMP_NAME);
@@ -789,15 +1095,25 @@ errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
 }
 
 /**
- *  @brief Flushes any applicable pending writes and unloads requested PNOR
- *         section from secure virtual address space
+ *  @brief Loads requested PNOR section to secure virtual address space
+ */
+errlHndl_t PNOR::loadSecureSection(const SectionId i_section)
+{
+    TRACFCOMP(g_trac_pnor, "loadSecureSection i_section = %i (%s)",
+              i_section,PNOR::SectionIdToString(i_section));
+
+    return loadUnloadSecureSection(i_section, PNOR::MSG_LOAD_SECTION);
+}
+
+/**
+ *  @brief Unloads requested PNOR section from secure virtual address space
  */
 errlHndl_t PNOR::unloadSecureSection(const SectionId i_section)
 {
-    // @TODO RTC 156118
-    // Replace with call to secure provider to unload the section
-    errlHndl_t pError=NULL;
-    return pError;
+    TRACFCOMP(g_trac_pnor, "unloadSecureSection i_section = %i (%s)",
+              i_section,PNOR::SectionIdToString(i_section));
+
+    return loadUnloadSecureSection(i_section, PNOR::MSG_UNLOAD_SECTION);
 }
 
 void SPnorRP::processLabOverride(
diff --git a/src/usr/pnor/spnorrp.H b/src/usr/pnor/spnorrp.H
index 889b70f43..878d69625 100644
--- a/src/usr/pnor/spnorrp.H
+++ b/src/usr/pnor/spnorrp.H
@@ -116,6 +116,13 @@ class SPnorRP
         uint8_t* secAddr;
         size_t textSize;
         size_t infoSize;
+        size_t refCount;
+        SHA512_t payloadTextHash;
+        LoadRecord()
+        :secAddr(nullptr), textSize(0), infoSize(0), refCount(0)
+        {
+            memset(&payloadTextHash[0], 0, SHA512_DIGEST_LENGTH);
+        }
     };
     std::map<PNOR::SectionId, LoadRecord*> iv_loadedSections;
 
@@ -127,11 +134,22 @@ class SPnorRP
 
     /**
      * @brief Load secure sections into temporary address space and verify them
-     * @param[in]  i_secId - PNOR section id to verify
-     * @param[in]  o_rec - Load record to store section information in
+     * @note The Load record is not only used for output of the section info
+     *       from the verifySections, but is also used as input via the
+     *       payloadTextHash field, which is used (if i_loadedPreviously is
+     *       true) to help determine if the PCR extend should be recalculated.
+     *
+     * @param[in]     i_secId - PNOR section id to verify
+     * @param[in]     i_loadedPreviously - indicates section has been securely
+     *                loaded previously
+     * @param[in/out] io_rec - Load record to store section information in
+     *                io_rec->payloadTextHash is used for comparision if
+     *                i_loadedPreviusly is true.
      * @return  uint64_t - Return code to pass back to message handler
      */
-    uint64_t verifySections(PNOR::SectionId i_id, LoadRecord* o_rec);
+    uint64_t verifySections(PNOR::SectionId i_id,
+                            bool i_loadedPreviously,
+                            LoadRecord* io_rec);
 
     /**
      * @brief  Message receiver for secure space
@@ -161,6 +179,16 @@ class SPnorRP
                                        uint64_t accessType) const;
 
     /**
+     * @brief  A wrapper for mm_remove_pages that encapsulates an error log
+     * @note   This is a special case of mm_remove_pages that makes use of
+     *         PAGE_REMOVAL_OPS "RELEASE" setting
+     *
+     * @param[in] i_va - virtual start address of pages to be removed
+     * @param[in] i_size - size of block to remove
+     */
+    errlHndl_t removePages(void* i_va, uint64_t i_size) const;
+
+    /**
      * @brief  Handles any additional section specific verification checks.
      * @param[in]  i_vaddr - vaddr of PNOR section to verify. Includes header
      *                       NULL will assert
diff --git a/src/usr/pnor/test/pnorrptest.H b/src/usr/pnor/test/pnorrptest.H
index d42ce93a5..c0267ebd2 100644
--- a/src/usr/pnor/test/pnorrptest.H
+++ b/src/usr/pnor/test/pnorrptest.H
@@ -830,12 +830,12 @@ class PnorRpTest : public CxxTest::TestSuite
         errlHndl_t pError=NULL;
         do {
 
-        if (!PNOR::isEnforcedSecureSection(PNOR::SBE_IPL))
+        if (!PNOR::isEnforcedSecureSection(PNOR::MEMD))
         {
             break;
         }
 
-        pError = PNOR::loadSecureSection(PNOR::SBE_IPL);
+        pError = PNOR::loadSecureSection(PNOR::MEMD);
         if(pError != NULL)
         {
             TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
@@ -844,7 +844,7 @@ class PnorRpTest : public CxxTest::TestSuite
             break;
         }
 
-        pError = PNOR::unloadSecureSection(PNOR::SBE_IPL);
+        pError = PNOR::unloadSecureSection(PNOR::MEMD);
         if(pError != NULL)
         {
             TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
@@ -853,6 +853,75 @@ class PnorRpTest : public CxxTest::TestSuite
             break;
         }
 
+        // try loading MEMD a few times
+        for (int i=0; i<10; i++)
+        {
+            pError = PNOR::loadSecureSection(PNOR::MEMD);
+            if(pError != nullptr)
+            {
+                TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
+                    "loadSecureSection returned an error on MEMD section load attempt %i",i);
+                break;
+            }
+        }
+        if (pError != nullptr)
+        {
+            ERRORLOG::errlCommit(pError,PNOR_COMP_ID);
+            break;
+        }
+
+
+        // try unloading MEMD the exact same number of times we loaded it
+        for (int i=0; i<10; i++)
+        {
+            pError = PNOR::unloadSecureSection(PNOR::MEMD);
+            if(pError != nullptr)
+            {
+                TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
+                    "loadSecureSection returned an error on MEMD section unload attempt %i", i);
+                break;
+            }
+        }
+        if (pError != nullptr)
+        {
+            ERRORLOG::errlCommit(pError,PNOR_COMP_ID);
+            break;
+        }
+
+        // TODO RTC 181272
+        // In order for the below test to work, we need to make sure that
+        // MEMD is never actually being loaded or unloaded during this test.
+        // Since all of the tests run in parallel, we can't really
+        // guarantee this completely if someone decides to write a test for
+        // MEMD, so ideally we would need some kind of mutex to prevent this.
+
+        // Try to unload the secure section one extra time
+        // We expect to see an error log
+        pError = PNOR::unloadSecureSection(PNOR::MEMD);
+        if(pError == nullptr)
+        {
+            TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
+                "unloadSecureSection failed to return error on extra invoke");
+            break;
+        }
+        else
+        {
+            if(pError->reasonCode() != PNOR::RC_EXTERNAL_ERROR ||
+               pError->moduleId() != PNOR::MOD_PNORRP_LOADUNLOADSECURESECTION)
+            {
+                ERRORLOG::errlCommit(pError, PNOR_COMP_ID);
+                TS_FAIL("PnorRpTest::test_loadUnloadSecureSection: "
+                    "unloadSecureSection return an unexpected error");
+                break;
+            }
+            else
+            {
+                // passed the test
+                delete pError;
+                pError = nullptr;
+            }
+        }
+
         } while (0);
 #endif
     }
diff --git a/src/usr/sbe/sbe_update.C b/src/usr/sbe/sbe_update.C
index 5054a5dfa..1a2cea5cb 100644
--- a/src/usr/sbe/sbe_update.C
+++ b/src/usr/sbe/sbe_update.C
@@ -1706,7 +1706,7 @@ namespace SBE
                            ERRL_GETPLID_SAFE(err));
                 break;
             }
-            
+
             bool l_bootSide0 = (l_bootside == SBE_SEEPROM0);
 
             TRACFCOMP( g_trac_sbe,INFO_MRK"updateSbeBootSeeprom(): set SBE boot side %d for proc=%.8X",
@@ -4226,16 +4226,6 @@ namespace SBE
                 break;
             }
 
-#ifndef CONFIG_SECUREBOOT
-            // @TODO RTC 157475
-            // UnloadSecureSection is not fully implemented so we do not attempt
-            // to pull the SBE partition back in after the initial time.
-            // NOTE: PNOR::flush(PNOR::HB_BOOTLOADER) is another thing that
-            //       could be flushed. It's only 20K, but it would be 5 pages
-            //       freed up.
-            PNOR::flush( PNOR::SBE_IPL );
-#endif
-
             // Unload PNOR sections from secure memory
 #ifdef CONFIG_SECUREBOOT
             err = unloadSecureSection(PNOR::SBE_IPL);
@@ -4251,14 +4241,17 @@ namespace SBE
                 TRACFCOMP( g_trac_sbe, ERR_MRK,"cleanupSbeImageVmmSpace() - Error from unloadSecureSection(PNOR::HB_BOOTLOADER)");
                 break;
             }
+
             err = unloadSecureSection(PNOR::HCODE);
             if (err)
             {
                 TRACFCOMP( g_trac_sbe, ERR_MRK,"cleanupSbeImageVmmSpace() - Error from unloadSecureSection(PNOR::HCODE)");
                 break;
             }
-
 #endif
+            PNOR::flush( PNOR::SBE_IPL );
+            PNOR::flush( PNOR::HB_BOOTLOADER );
+            PNOR::flush( PNOR::HCODE );
 
         }while(0);
 
diff --git a/src/usr/testcore/rtloader/loader.H b/src/usr/testcore/rtloader/loader.H
index 60056c633..62e251370 100644
--- a/src/usr/testcore/rtloader/loader.H
+++ b/src/usr/testcore/rtloader/loader.H
@@ -62,10 +62,6 @@ class RuntimeLoaderTest : public CxxTest::TestSuite
             errlHndl_t l_errl = nullptr;
 
 #ifdef CONFIG_SECUREBOOT
-            // load secure section
-            // TODO RTC: 157475 Since this is a test case and unload is
-            // merely a stub function at this point in time, add a call
-            // to unload later when the aforementioned story is implemented.
             l_errl = loadSecureSection(PNOR::HB_RUNTIME);
             if(l_errl)
             {
@@ -178,6 +174,18 @@ class RuntimeLoaderTest : public CxxTest::TestSuite
 
             mm_set_permission(imageArea, imageSize, WRITABLE);
             free(imageArea);
+
+#ifdef CONFIG_SECUREBOOT
+            l_errl = unloadSecureSection(PNOR::HB_RUNTIME);
+            if(l_errl)
+            {
+                TS_FAIL("Could not securely load runtime section.");
+                delete l_errl;
+                l_errl = nullptr;
+                return;
+            }
+#endif
+
         }
 
     private:
diff --git a/src/usr/util/utillidmgr.C b/src/usr/util/utillidmgr.C
index a84f31e88..f8ef376e0 100644
--- a/src/usr/util/utillidmgr.C
+++ b/src/usr/util/utillidmgr.C
@@ -773,8 +773,9 @@ errlHndl_t UtilLidMgr::cleanup()
 #ifdef CONFIG_SECUREBOOT
         // If in SECUREBOOT the lid could be securely signed in PNOR (like OCC)
         // If so, unload it securely
-        // NOTE:  It is safe to unload it even if it was unloaded before
-        if (iv_lidPnorInfo.secure)
+        bool l_doUnload = (iv_lidPnorInfo.size != 0);
+
+        if (iv_lidPnorInfo.secure && l_doUnload)
         {
             l_err = PNOR::unloadSecureSection(iv_lidPnorInfo.id);
 
@@ -785,6 +786,10 @@ errlHndl_t UtilLidMgr::cleanup()
                                "unloading module : %s (id=0x%X)",
                                iv_lidPnorInfo.id, iv_lidFileName, iv_lidId);
             }
+            else
+            {
+                iv_lidPnorInfo.size = 0;
+            }
         }
 #endif