Handle early life PNOR fails in HBRT instead of hanging

A hang happens when RtPNOR code creates an error log
while it still hasn't initialized completely.  Error log code
calls PNOR code that hasn't completed initialization yet.
The fix is to assert in HBRT and by the time HBRT gets restarted,
PNOR should be present and accessible.

Change-Id: I24a4046be9da921933e7ca9005088945a0c25cfa
RTC:189291
CQ:SW423599
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/56802
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 104 insertions, 22 deletions

diff --git a/src/usr/pnor/pnor_common.C b/src/usr/pnor/pnor_common.C
index c64236709..6bf7cc480 100644
--- a/src/usr/pnor/pnor_common.C
+++ b/src/usr/pnor/pnor_common.C
@@ -134,7 +134,8 @@ void  PNOR::physicalToMmioOffset(uint64_t  i_hbbAddress,
  *  @brief: parse the TOCs read from memory and store section information
  *          from one of the verified TOCs
  */
-errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC)
+errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC,
+                           bool i_pnorInitialized )
 {
     TRACUCOMP(g_trac_pnor,"PNOR::parseTOC>");
     errlHndl_t l_errhdl = NULL;
@@ -153,6 +154,11 @@ errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC)
         if(l_errCode != NO_ERROR)
         {
             TRACFCOMP(g_trac_pnor, "Null TOC Buffer found while checking TOC" );
+
+            // prevent hang between ErrlManager and Pnor
+            assert(i_pnorInitialized,
+                   "Null TOC Buffer found while checking TOC"
+                   " during pnor initialization");
             /*@
             * @errortype
             * @moduleid PNOR::MOD_PNORRP_READTOC
@@ -185,6 +191,12 @@ errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC)
         if (l_errCode != NO_ERROR)
         {
             TRACFCOMP(g_trac_pnor, "PNOR::parseTOC Checksum error in TOC's header");
+
+            // prevent hang between ErrlManager and Pnor
+            assert(i_pnorInitialized,
+                      "PNOR::parseTOC Found checksum error in TOC's header"
+                      " during pnor initialization");
+
             /* @errortype
             * @moduleid PNOR::MOD_PNORRP_READTOC
             * @reasoncode PNOR::RC_TOC_HDR_CHECKSUM_ERR
@@ -212,7 +224,12 @@ errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC)
         PNOR::checkHeader(l_ffs_hdr, l_errCode);
         if(l_errCode != NO_ERROR)
         {
-            TRACFCOMP(g_trac_pnor, "PNOR::parseTOC Error found parsing hdr of TOC " );
+            TRACFCOMP(g_trac_pnor, "PNOR::parseTOC Error found parsing hdr of TOC" );
+
+            // prevent hang between ErrlManager and Pnor
+            assert(i_pnorInitialized,
+                      "PNOR::parseTOC Error found parsing hdr of TOC"
+                      " during pnor initialization");
             /* @errortype
             * @moduleid PNOR::MOD_PNORRP_READTOC
             * @reasoncode PNOR::RC_BAD_TOC_HEADER
@@ -253,6 +270,12 @@ errlHndl_t PNOR::parseTOC( uint8_t* i_tocBuffer,SectionData_t * o_TOC)
         {
             TRACFCOMP(g_trac_pnor, "PNOR::parseTOC parseEntries returned an error code");
             o_TOC = NULL;
+
+            // prevent hang between ErrlManager and Pnor
+            assert(i_pnorInitialized,
+                    "PNOR::parseTOC parseEntries returned an error code"
+                    " during pnor initialization");
+
             /* @errortype
             * @moduleid PNOR::MOD_PNORRP_READTOC
             * @reasoncode PNOR::RC_PNOR_PARSE_ENTRIES_ERR
diff --git a/src/usr/pnor/pnor_common.H b/src/usr/pnor/pnor_common.H
index 8ada2c03b..bd9ae5fb5 100644
--- a/src/usr/pnor/pnor_common.H
+++ b/src/usr/pnor/pnor_common.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2014,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2014,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -39,10 +39,12 @@ namespace PNOR {
      * @brief parse the TOC read from memory and store section
      *        information if the TOC is valid
      * @param[in] i_tocBuffer      Pointer to buffer containing a TOC
+     * @param[in] i_pnorInitialized PNOR::init() has completed
      * @param[out] o_TOC            sectionInformation of the TOC used
      * @return Error
      */
-    errlHndl_t parseTOC(uint8_t* i_tocBuffer, PNOR::SectionData_t * o_TOC);
+    errlHndl_t parseTOC(uint8_t* i_tocBuffer, PNOR::SectionData_t * o_TOC,
+                        bool i_pnorInitialized = true);
 
     /**
      *  @brief: determine the physical offset of ffs entry
diff --git a/src/usr/pnor/runtime/rt_pnor.C b/src/usr/pnor/runtime/rt_pnor.C
index ba23cecb5..f3e969f2e 100644
--- a/src/usr/pnor/runtime/rt_pnor.C
+++ b/src/usr/pnor/runtime/rt_pnor.C
@@ -128,15 +128,25 @@ void RtPnor::init(errlHndl_t &io_taskRetErrl)
     {
       TRACFCOMP(g_trac_pnor, "Rtnor: failed to read FIRDATA section" );
     }
+    else
+    {
+      Singleton<RtPnor>::instance().setInitialized(true);
+    }
 
     TRACFCOMP(g_trac_pnor, EXIT_MRK"RtPnor::init()");
 }
 
 /**************************************************************/
+void RtPnor::setInitialized(bool i_initialized)
+{
+  iv_initialized = i_initialized;
+}
+
+/**************************************************************/
 errlHndl_t RtPnor::getSectionInfo(PNOR::SectionId i_section,
                               PNOR::SectionInfo_t& o_info)
 {
-    TRACFCOMP(g_trac_pnor, ENTER_MRK"RtPnor::getSectionInfo %d", i_section);
+    TRACFCOMP(g_trac_pnor, ENTER_MRK"RtPnor::getSectionInfo %d, initialized = %d", i_section, iv_initialized?1:0);
     errlHndl_t l_err = nullptr;
     do
     {
@@ -199,6 +209,11 @@ errlHndl_t RtPnor::getSectionInfo(PNOR::SectionId i_section,
         {
             TRACFCOMP(g_trac_pnor,"RtPnor::getSectionInfo: Section %d"
                     " size is 0", static_cast<int>(i_section));
+
+            // prevent hang between ErrlManager and rt_pnor
+            assert(iv_initialized,
+                   "RtPnor::getSectionInfo: Section size 0 returned"
+                   " before completing PNOR initialization");
             /*@
              * @errortype
              * @moduleid    PNOR::MOD_RTPNOR_GETSECTIONINFO
@@ -366,6 +381,7 @@ errlHndl_t RtPnor::flush( PNOR::SectionId i_section)
 /*******Protected Methods**************/
 RtPnor::RtPnor()
 {
+    iv_initialized = false;
     do {
         errlHndl_t l_err = getMasterProcId();
         if (l_err)
@@ -430,6 +446,12 @@ errlHndl_t RtPnor::readFromDevice (uint64_t i_procId,
                         " failed proc:%d, part:%s, offset:0x%X, size:0x%X,"
                         " dataPt:0x%X, rc:%d", i_procId, l_partitionName,
                         l_offset, l_readSize, l_dataToRead, l_rc);
+
+                // prevent hang between ErrlManager and rt_pnor
+                assert(iv_initialized,
+                      "RtPnor::readFromDevice: pnor_read returned an error"
+                      " during initialization");
+
                 /*@
                  * @errortype
                  * @moduleid            PNOR::MOD_RTPNOR_READFROMDEVICE
@@ -464,6 +486,11 @@ errlHndl_t RtPnor::readFromDevice (uint64_t i_procId,
                 }
                 else // everything else should have a known size
                 {
+                  // prevent hang between ErrlManager and rt_pnor
+                  assert(iv_initialized,
+                      "RtPnor::readFromDevice: pnor_read failed to read "
+                      "expected amount before rt_pnor initialization");
+
                     /*@
                      * @errortype
                      * @moduleid            PNOR::MOD_RTPNOR_READFROMDEVICE
@@ -491,20 +518,25 @@ errlHndl_t RtPnor::readFromDevice (uint64_t i_procId,
         {
             TRACFCOMP(g_trac_pnor,"RtPnor::readFromDevice: This version of"
                     " OPAL does not support pnor_read");
-                /*@
-                 * @errortype
-                 * @moduleid           PNOR::MOD_RTPNOR_READFROMDEVICE
-                 * @reasoncode         PNOR::RC_PNOR_READ_NOT_SUPPORTED
-                 * @devdesc            g_hostInterfaces->pnor_read not supported
-                 * @custdesc           Error accessing system firmware flash
-                 */
-                //@todo Add PNOR callout RTC:116145
-                l_err = new ERRORLOG::ErrlEntry(
-                                 ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                 PNOR::MOD_RTPNOR_READFROMDEVICE,
-                                 PNOR::RC_PNOR_READ_NOT_SUPPORTED,
-                                 0,0,true);
-                break;
+
+            // prevent hang between ErrlManager and rt_pnor
+            assert(iv_initialized,
+                      "RtPnor::readFromDevice: OPAL version does NOT support"
+                      "pnor_read during initialization");
+            /*@
+             * @errortype
+             * @moduleid           PNOR::MOD_RTPNOR_READFROMDEVICE
+             * @reasoncode         PNOR::RC_PNOR_READ_NOT_SUPPORTED
+             * @devdesc            g_hostInterfaces->pnor_read not supported
+             * @custdesc           Error accessing system firmware flash
+             */
+            //@todo Add PNOR callout RTC:116145
+            l_err = new ERRORLOG::ErrlEntry(
+                             ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                             PNOR::MOD_RTPNOR_READFROMDEVICE,
+                             PNOR::RC_PNOR_READ_NOT_SUPPORTED,
+                             0,0,true);
+            break;
         }
         // remove the ECC data
         if( i_ecc )
@@ -524,6 +556,11 @@ errlHndl_t RtPnor::readFromDevice (uint64_t i_procId,
                 TRACFCOMP(g_trac_pnor,"RtPnor::readFromDevice>"
                     " Uncorrectable ECC error : chip=%d,offset=0x%.X",
                     i_procId, i_offset );
+
+                // prevent hang between ErrlManager and rt_pnor
+                assert(iv_initialized,
+                      "RtPnor::readFromDevice: UNCORRECTABLE_ECC encountered"
+                      " during initialization");
                 /*@
                  * @errortype
                  * @moduleid    PNOR::MOD_RTPNOR_READFROMDEVICE
@@ -556,6 +593,11 @@ errlHndl_t RtPnor::readFromDevice (uint64_t i_procId,
                         TRACFCOMP(g_trac_pnor, "RtPnor::readFromDevice> Error"
                         " writing corrected data back to device");
 
+                        // prevent hang between ErrlManager and rt_pnor
+                        assert(iv_initialized,
+                            "RtPnor::readFromDevice: pnor_write returned an"
+                            " error during initialization");
+
                         /*@
                          * @errortype
                          * @moduleid   PNOR::MOD_RTPNOR_READFROMDEVICE
@@ -712,7 +754,7 @@ errlHndl_t RtPnor::readTOC ()
             //Pass along TOC buffer to be parsed, parseTOC will parse through
             // the buffer and store needed information in iv_TOC
             // Note: that Opal should always return a valid TOC
-            l_err = PNOR::parseTOC(l_toc0Buffer, iv_TOC);
+            l_err = PNOR::parseTOC(l_toc0Buffer, iv_TOC, iv_initialized);
             if (l_err)
             {
                 TRACFCOMP(g_trac_pnor, "RtPnor::readTOC: parseTOC failed");
@@ -912,7 +954,6 @@ void initPnor()
         errlCommit (l_errl, PNOR_COMP_ID);
       }
     }
-
     TRACFCOMP(g_trac_pnor, EXIT_MRK"initPnor");
 }
 
diff --git a/src/usr/pnor/runtime/rt_pnor.H b/src/usr/pnor/runtime/rt_pnor.H
index ab612bebd..7a0254be9 100644
--- a/src/usr/pnor/runtime/rt_pnor.H
+++ b/src/usr/pnor/runtime/rt_pnor.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2014,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2014,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -96,6 +96,19 @@ class RtPnor
          */
         errlHndl_t getPnorInfo( PNOR::PnorInfo_t& o_pnorInfo );
 
+
+        /**
+         * @brief  Has RtPnor::init() successfully completed?
+         * @return true if init() finished successfully, otherwise false
+         */
+        bool isPnorInitialized() { return iv_initialized; }
+
+        /**
+         * @brief  Accessor function for iv_initialized.
+         *         true = RtPnor::init() finished successfully
+         */
+        void setInitialized(bool i_initialized);
+
     protected:
        /**
         * @brief Constructor
@@ -176,6 +189,9 @@ class RtPnor
         // Cached master Proc Id
         static uint64_t iv_masterProcId;
 
+        // is RT pnor initialized?
+        bool iv_initialized;
+
         //allow testcases to see inside the class
         friend class PnorRtTest;