PNORDD Improvement to handle LPC Slave Errors

This commit contains improvements to PNORRP and PNORDD error
callouts.  It also includes looking for errors in the LPC slave
status register and specifc errors in the SFC status register.
It also adds a FFDC function that dumps registers and adds them
to the error log.

Change-Id: I5fee0a65063999d15fb3d9bbd4ca64fcc409de4d
RTC: 62718
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/8745
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

5 files changed, 679 insertions, 49 deletions

diff --git a/src/include/usr/pnor/pnor_reasoncodes.H b/src/include/usr/pnor/pnor_reasoncodes.H
index 482d55f90..a643a1fe6 100644
--- a/src/include/usr/pnor/pnor_reasoncodes.H
+++ b/src/include/usr/pnor/pnor_reasoncodes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -50,6 +50,7 @@ namespace PNOR
         MOD_PNORDD_WRITELPC             = 0x19, /**< pnordd.C : PnorDD::writeLPC */
         MOD_PNORDD_ERASEFLASH           = 0x1A, /**< pnordd.C : PnorDD::eraseFlash */
         MOD_PNORDD_MICRONFLAGSTATUS     = 0x1B, /**< pnordd.C : PnorDD::micronFlagStatus */
+        MOD_PNORDD_CHECKFORERRORS       = 0x1D, /**< pnordd.C : PnorDD::checkForErrors */
 
     };
 
@@ -75,6 +76,7 @@ namespace PNOR
         RC_UNSUPPORTED_SFCRANGE      = PNOR_COMP_ID | 0x0E,
         //termination_rc
         RC_ECC_UE                    = PNOR_COMP_ID | 0x0F,
+        RC_ERROR_IN_STATUS_REG       = PNOR_COMP_ID | 0x10,
     };
 };
 
diff --git a/src/usr/pnor/pnordd.C b/src/usr/pnor/pnordd.C
index 7a20b892e..e54cb401a 100644
--- a/src/usr/pnor/pnordd.C
+++ b/src/usr/pnor/pnordd.C
@@ -36,8 +36,9 @@
 #include <devicefw/driverif.H>
 #include <trace/interface.H>
 #include <errl/errlentry.H>
-#include <targeting/common/targetservice.H>
 #include <errl/errlmanager.H>
+#include <errl/errludlogregister.H>
+#include <targeting/common/targetservice.H>
 #include "pnordd.H"
 #include <pnor/pnorif.H>
 #include <pnor/pnor_reasoncodes.H>
@@ -366,6 +367,7 @@ PnorDD::PnorDD( PnorMode_t i_mode,
                 uint64_t i_fakeStart,
                 uint64_t i_fakeSize )
 : iv_mode(i_mode)
+, iv_ffdc_active(false)
 {
     iv_erasesize_bytes = ERASESIZE_BYTES_DEFAULT;
 
@@ -545,8 +547,7 @@ void PnorDD::sfcInit( )
             }
             else
             {
-                TRACFCOMP( g_trac_pnor,  ERR_MRK
-                           "PnorDD::sfcInit> Unsupported NOR type detected : cv_nor_chipid=%.4X",
+                TRACFCOMP( g_trac_pnor,  ERR_MRK"PnorDD::sfcInit> Unsupported NOR type detected : cv_nor_chipid=%.4X. Calling doShutdown(PNOR::RC_UNSUPPORTED_HARDWARE)",
                            cv_nor_chipid );
 
                 //Shutdown if we detect unsupported Hardware
@@ -566,6 +567,8 @@ void PnorDD::sfcInit( )
 
     if( l_err )
     {
+        TRACFCOMP( g_trac_pnor,  ERR_MRK
+                   "PnorDD::sfcInit> Committing error log");
         errlCommit(l_err,PNOR_COMP_ID);
     }
 }
@@ -615,10 +618,14 @@ errlHndl_t PnorDD::writeRegSfc(SfcRange i_range,
             lpc_addr = LPC_SFC_CMDBUF_OFFSET | i_addr;
             break;
         }
+    case SFC_LPC_SPACE:
+        {
+            lpc_addr = LPCHC_FW_SPACE | i_addr;
+            break;
+        }
     default:
         {
-            TRACFCOMP(g_trac_pnor, ERR_MRK
-                      "PnorDD::writeRegSfc> Unsupported SFC Address Range: i_range=0x%.16X, i_addr=0x%.8X",
+            TRACFCOMP(g_trac_pnor, ERR_MRK"PnorDD::writeRegSfc> Unsupported SFC Address Range: i_range=0x%.16X, i_addr=0x%.8X. Calling doShutdown(PNOR::RC_UNSUPPORTED_SFCRANGE)",
                       i_range, i_addr);
 
             //Can't function without PNOR, initiate shutdown.
@@ -661,10 +668,14 @@ errlHndl_t PnorDD::readRegSfc(SfcRange i_range,
             lpc_addr = LPC_SFC_CMDBUF_OFFSET | i_addr;
             break;
         }
+    case SFC_LPC_SPACE:
+        {
+            lpc_addr = LPCHC_FW_SPACE | i_addr;
+            break;
+        }
     default:
         {
-            TRACFCOMP(g_trac_pnor, ERR_MRK
-                      "PnorDD::readRegSfc> Unsupported SFC Address Range: i_range=0x%.16X, i_addr=0x%.8X",
+            TRACFCOMP(g_trac_pnor, ERR_MRK"PnorDD::readRegSfc> Unsupported SFC Address Range: i_range=0x%.16X, i_addr=0x%.8X. Calling doShutdown(PNOR::RC_UNSUPPORTED_SFCRANGE)",
                       i_range, i_addr);
 
             //Can't function without PNOR, initiate shutdown.
@@ -702,7 +713,9 @@ errlHndl_t PnorDD::pollSfcOpComplete(uint64_t i_pollTime)
                                sfc_stat.data32);
             if(l_err) { break; }
 
-            if( sfc_stat.done == 1 )
+            if( ( sfc_stat.done == 1 ) ||
+                ( sfc_stat.timeout == 1 ) ||
+                ( sfc_stat.illegal == 1 ) )
             {
                 break;
             }
@@ -716,8 +729,12 @@ errlHndl_t PnorDD::pollSfcOpComplete(uint64_t i_pollTime)
         }
         if( l_err ) { break; }
 
-        // check for errors or timeout
-        // TODO: RTC:62718 What errors do we check?
+
+        l_err = checkForErrors();
+        if( l_err ) { break; }
+
+
+        // If no errors AND done bit not set, call out undefined error
         if( (sfc_stat.done == 0) )
         {
             TRACFCOMP(g_trac_pnor,
@@ -741,6 +758,14 @@ errlHndl_t PnorDD::pollSfcOpComplete(uint64_t i_pollTime)
                                                                  poll_time),
                                             TWO_UINT32_TO_UINT64(sfc_stat.data32,0));
 
+            // Limited in callout: no PNOR target, so calling out processor
+            l_err->addHwCallout(
+                            TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                            HWAS::SRCI_PRIORITY_HIGH,
+                            HWAS::NO_DECONFIG,
+                            HWAS::GARD_NULL );
+
+            addFFDCRegisters(l_err);
             l_err->collectTrace(PNOR_COMP_NAME);
             l_err->collectTrace(XSCOM_COMP_NAME);
             //@todo (RTC:37744) - Any cleanup or recovery needed?
@@ -755,9 +780,327 @@ errlHndl_t PnorDD::pollSfcOpComplete(uint64_t i_pollTime)
 }
 
 /**
+ * @brief Check For Errors in Status Registers
+ */
+errlHndl_t PnorDD::checkForErrors( void )
+{
+
+    errlHndl_t l_err = NULL;
+    bool errorFound = false;
+
+    // Default status values in case we fail in reading the registers
+    LpcSlaveStatReg_t lpc_slave_stat;
+    lpc_slave_stat.data32 = 0xDEADBEEF;
+    SfcStatReg_t sfc_stat;
+    sfc_stat.data32 = 0xDEADBEEF;
+
+
+    do {
+
+        // First Read LPC Slave Status Register
+        l_err = readRegSfc(SFC_LPC_SPACE,
+                           LPC_SLAVE_REG_STATUS,
+                           lpc_slave_stat.data32);
+
+        // If we can't read status register, exit out
+        if( l_err ) { break; }
+
+        TRACDCOMP( g_trac_pnor, INFO_MRK"PnorDD::checkForErrors> LPC Slave status reg: 0x%08llx",
+                   lpc_slave_stat.data32);
+
+        if( 1 == lpc_slave_stat.lbusparityerror )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LPC Slave Local Bus Parity Error: status reg: 0x%08llx",
+                       lpc_slave_stat.data32);
+        }
+
+        if( 0 != lpc_slave_stat.lbus2opberr )
+        {
+            errorFound = true;
+
+            if ( LBUS2OPB_ADDR_PARITY_ERR == lpc_slave_stat.lbus2opberr )
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB Address Parity Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+
+            }
+
+            else if ( LBUS2OPB_INVALID_SELECT_ERR == lpc_slave_stat.lbus2opberr)
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB Invalid Select Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+
+            }
+            else if ( LBUS2OPB_DATA_PARITY_ERR == lpc_slave_stat.lbus2opberr )
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB Data Parity Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+
+            }
+            else if ( LBUS2OPB_MONITOR_ERR == lpc_slave_stat.lbus2opberr )
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB Monitor Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+
+            }
+
+            else if ( LBUS2OPB_TIMEOUT_ERR == lpc_slave_stat.lbus2opberr )
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB Timeout Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+
+            }
+            else
+            {
+                TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> LBUS2OPB UNKNOWN Error: LPC Slave status reg: 0x%08llx",
+                           lpc_slave_stat.data32);
+            }
+
+        }
+
+        // Second Read SFC and check for error bits
+        l_err = readRegSfc(SFC_CMD_SPACE,
+                           SFC_REG_STATUS,
+                           sfc_stat.data32);
+
+        // If we can't read status register, exit out
+        if( l_err ) { break; }
+
+        TRACDCOMP( g_trac_pnor, INFO_MRK"PnorDD::checkForErrors> SFC status reg(0x%X): 0x%08llx",
+                   SFC_CMD_SPACE|SFC_REG_STATUS,sfc_stat.data32);
+
+        if( 1 == sfc_stat.eccerrcntr )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Threshold of SRAM ECC Errors Reached: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+        if( 1 == sfc_stat.eccues )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> SRAM Command Uncorrectable ECC Error: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+        if( 1 == sfc_stat.illegal )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Previous Operation was Illegal: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+        if( 1 == sfc_stat.eccerrcntn )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Threshold for Flash ECC Errors Reached: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+        if( 1 == sfc_stat.eccuen )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Flash Command Uncorrectable ECC Error: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+        if( 1 == sfc_stat.timeout )
+        {
+            errorFound = true;
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Timeout: SFC status reg: 0x%08llx",
+                       sfc_stat.data32);
+        }
+
+    }while(0);
+
+
+    // If there is any error create an error log
+    if ( errorFound )
+    {
+        // If we failed on a register read above, but still found an error,
+        // delete register read error log and create an original error log
+        // for the found error
+        if ( l_err )
+        {
+            TRACFCOMP( g_trac_pnor, ERR_MRK"PnorDD::checkForErrors> Deleting register read error. Returning error created for the found error");
+            delete l_err;
+        }
+
+
+        /*@
+         * @errortype
+         * @moduleid     PNOR::MOD_PNORDD_CHECKFORERRORS
+         * @reasoncode   PNOR::RC_ERROR_IN_STATUS_REG
+         * @userdata1[0:31]  SFC Status Register
+         * @userdata1[32:63] <unused>
+         * @userdata2[0:31]  LPC Slave Status Register
+         * @userdata2[32:63] <unused>
+         * @devdesc      PnorDD::checkForErrors> Error(s) found in SFC
+         *               and/or LPC Slave Status Registers
+         */
+        l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                        PNOR::MOD_PNORDD_CHECKFORERRORS,
+                                        PNOR::RC_ERROR_IN_STATUS_REG,
+                                        TWO_UINT32_TO_UINT64(
+                                                   sfc_stat.data32,
+                                                   0),
+                                        TWO_UINT32_TO_UINT64(
+                                                   lpc_slave_stat.data32,
+                                                   0));
+
+        // Limited in callout: no PNOR target, so calling out processor
+        l_err->addHwCallout(
+                        TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                        HWAS::SRCI_PRIORITY_HIGH,
+                        HWAS::NO_DECONFIG,
+                        HWAS::GARD_NULL );
+
+        addFFDCRegisters(l_err);
+        l_err->collectTrace(PNOR_COMP_NAME);
+    }
+
+    return l_err;
+
+}
+
+
+/**
+ * @brief Add Error Registers to an existing Error Log
+ */
+void PnorDD::addFFDCRegisters(errlHndl_t & io_errl)
+{
+
+    errlHndl_t tmp_err = NULL;
+    uint32_t data32 = 0;
+    uint64_t data64 = 0;
+    size_t size64 = sizeof(data64);
+    size_t size32 = sizeof(data32);
+
+    // check iv_ffdc_active to avoid infinite loops
+    if ( iv_ffdc_active == false )
+    {
+        iv_ffdc_active = true;
+
+        TRACFCOMP( g_trac_pnor, "PnorDD::addFFDCRegisters> adding FFDC to Error Log EID=0x%X, PLID=0x%X",
+                   io_errl->eid(), io_errl->plid() );
+
+        ERRORLOG::ErrlUserDetailsLogRegister
+                  l_eud(TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL);
+
+        do {
+
+            // Add ECCB Status Register
+            TARGETING::Target* scom_target =
+                TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL;
+
+            tmp_err = deviceOp( DeviceFW::READ,
+                                scom_target,
+                                &(data64),
+                                size64,
+                                DEVICE_SCOM_ADDRESS(ECCB_STAT_REG) );
+
+            if( tmp_err )
+            {
+                delete tmp_err;
+                TRACFCOMP( g_trac_pnor, "PnorDD::addFFDCRegisters> Fail reading ECCB_STAT_REG");
+            }
+            else
+            {
+                l_eud.addDataBuffer(&data64, size64,
+                                    DEVICE_SCOM_ADDRESS(ECCB_STAT_REG));
+            }
+
+            // Add LPC Slave Status Register
+            LpcSlaveStatReg_t lpc_slave_stat;
+
+            tmp_err = readRegSfc(SFC_LPC_SPACE,
+                                 LPC_SLAVE_REG_STATUS,
+                                 lpc_slave_stat.data32);
+
+            if ( tmp_err )
+            {
+                delete tmp_err;
+                TRACFCOMP( g_trac_pnor, "PnorDD::addFFDCRegisters> Fail reading LPC Slave Status Register");
+            }
+            else
+            {
+                l_eud.addDataBuffer(&lpc_slave_stat.data32, size32,
+                      DEVICE_SCOM_ADDRESS( LPCHC_FW_SPACE |
+                                           LPC_SLAVE_REG_STATUS));
+            }
+
+            // Add SFC Registers
+            uint32_t sfc_regs[] = {
+                SFC_REG_STATUS,
+                SFC_REG_CONF,
+                SFC_REG_CMD,
+                SFC_REG_ADR,
+                SFC_REG_ERASMS,
+                SFC_REG_ERASLGS,
+                SFC_REG_CONF4,
+                SFC_REG_CONF5,
+                SFC_REG_ADRCBF,
+                SFC_REG_ADRCMF,
+                SFC_REG_OADRNB,
+                SFC_REG_OADRNS,
+                SFC_REG_CHIPIDCONF,
+                SFC_REG_ERRCONF,
+                SFC_REG_ERRTAG,
+                SFC_REG_ERROFF,
+                SFC_REG_ERRSYN,
+                SFC_REG_ERRDATH,
+                SFC_REG_ERRDATL,
+                SFC_REG_ERRCNT,
+                SFC_REG_CLRCNT,
+                SFC_REG_ERRINJ,
+                SFC_REG_PROTA,
+                SFC_REG_PROTM,
+                SFC_REG_ECCADDR,
+                SFC_REG_ECCRNG,
+                SFC_REG_ERRORS,
+                SFC_REG_INTMSK,
+                SFC_REG_INTENM,
+                SFC_REG_CONF2,
+                SFC_REG_CONF3
+            };
+
+
+            for( size_t x=0; x<(sizeof(sfc_regs)/sizeof(sfc_regs[0])); x++ )
+            {
+                tmp_err = readRegSfc( SFC_CMD_SPACE,
+                                      sfc_regs[x],
+                                      data32 );
+
+                if( tmp_err )
+                {
+                    delete tmp_err;
+                }
+                else
+                {
+                    l_eud.addDataBuffer(&data32, size32,
+                          DEVICE_SCOM_ADDRESS(LPC_SFC_CMDREG_OFFSET
+                                              | sfc_regs[x]));
+                }
+            }
+
+        }while(0);
+
+
+        l_eud.addToLog(io_errl);
+
+        // reset FFDC active flag
+        iv_ffdc_active = false;
+    }
+
+    return;
+}
+
+
+/**
  * @brief Check flag status bit on Micron NOR chips
  *        The current version of Micron parts require the Flag
- *        Status register be read after a read or erase operation,
+ *        Status register be read after a write or erase operation,
  *        otherwise all future operations won't work..
  */
 errlHndl_t PnorDD::micronFlagStatus(uint64_t i_pollTime)
@@ -844,8 +1187,14 @@ errlHndl_t PnorDD::micronFlagStatus(uint64_t i_pollTime)
                                                                  poll_time),
                                             TWO_UINT32_TO_UINT64(opStatus,0));
 
-            l_err->collectTrace(PNOR_COMP_NAME);
-            l_err->collectTrace(XSCOM_COMP_NAME);
+            // Limited in callout: no PNOR target, so calling out processor
+            l_err->addHwCallout(
+                            TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                            HWAS::SRCI_PRIORITY_HIGH,
+                            HWAS::NO_DECONFIG,
+                            HWAS::GARD_NULL );
+
+            addFFDCRegisters(l_err);
 
             //Erase & Program error bits are sticky, so they need to be cleared.
 
@@ -859,9 +1208,9 @@ errlHndl_t PnorDD::micronFlagStatus(uint64_t i_pollTime)
             tmp_err = writeRegSfc(SFC_CMD_SPACE,
                                 SFC_REG_CHIPIDCONF,
                                 confData);
-            if(tmp_err) {
-                //commit this error and return the original
-                ERRORLOG::errlCommit(tmp_err,PNOR_COMP_ID);
+            if(tmp_err)
+            {
+                delete tmp_err;
             }
 
             //Issue Get Chip ID command (clearing flag status)
@@ -869,10 +1218,23 @@ errlHndl_t PnorDD::micronFlagStatus(uint64_t i_pollTime)
             sfc_cmd.opcode = SFC_OP_CHIPID;
             sfc_cmd.length = 0;
 
-            l_err = writeRegSfc(SFC_CMD_SPACE,
-                                SFC_REG_CMD,
-                                sfc_cmd.data32);
-            if(l_err) { break; }
+            tmp_err = writeRegSfc(SFC_CMD_SPACE,
+                                  SFC_REG_CMD,
+                                  sfc_cmd.data32);
+            if(tmp_err)
+            {
+                delete tmp_err;
+            }
+
+            //Poll for complete status
+            tmp_err = pollSfcOpComplete();
+            if(tmp_err)
+            {
+                delete tmp_err;
+            }
+
+            l_err->collectTrace(PNOR_COMP_NAME);
+            l_err->collectTrace(XSCOM_COMP_NAME);
 
             break;
         }
@@ -1141,8 +1503,7 @@ errlHndl_t PnorDD::bufferedSfcRead(uint32_t i_addr,
             }
         default:
             {
-                TRACFCOMP(g_trac_pnor, ERR_MRK
-                          "PnorDD::bufferedSfcRead> Unsupported mode: iv_mode=0x%.16X, i_addr=0x%.8X",
+                TRACFCOMP(g_trac_pnor, ERR_MRK"PnorDD::bufferedSfcRead> Unsupported mode: iv_mode=0x%.16X, i_addr=0x%.8X. Calling doShutdown(PNOR::RC_UNSUPPORTED_MODE)",
                           iv_mode, i_addr);
 
                 //Can't function without PNOR, initiate shutdown.
@@ -1215,8 +1576,7 @@ errlHndl_t PnorDD::bufferedSfcWrite(uint32_t i_addr,
             }
         default:
             {
-                TRACFCOMP(g_trac_pnor, ERR_MRK
-                          "PnorDD::bufferedSfcWrite> Unsupported mode: iv_mode=0x%.16X, i_addr=0x%.8X",
+                TRACFCOMP(g_trac_pnor, ERR_MRK"PnorDD::bufferedSfcWrite> Unsupported mode: iv_mode=0x%.16X, i_addr=0x%.8X. Calling doShutdown(PNOR::RC_UNSUPPORTED_MODE)",
                           iv_mode, i_addr);
 
                 //Can't function without PNOR, initiate shutdown.
@@ -1317,6 +1677,7 @@ errlHndl_t PnorDD::readLPC(uint32_t i_addr,
         StatusReg_t eccb_stat;
         uint64_t poll_time = 0;
         uint64_t loop = 0;
+
         while( poll_time < ECCB_POLL_TIME_NS )
         {
             l_err = deviceOp( DeviceFW::READ,
@@ -1341,7 +1702,7 @@ errlHndl_t PnorDD::readLPC(uint32_t i_addr,
         if( l_err ) { break; }
 
         // check for errors or timeout
-        if( (eccb_stat.data64 & LPC_STAT_REG_ERROR_MASK)
+        if( (eccb_stat.data64 & ECCB_LPC_STAT_REG_ERROR_MASK)
             || (eccb_stat.op_done == 0) )
         {
             TRACFCOMP(g_trac_pnor, "PnorDD::readLPC> Error or timeout from LPC Status Register : i_addr=0x%.8X, status=0x%.16X", i_addr, eccb_stat.data64 );
@@ -1361,6 +1722,15 @@ errlHndl_t PnorDD::readLPC(uint32_t i_addr,
                                             PNOR::RC_LPC_ERROR,
                                             TWO_UINT32_TO_UINT64(i_addr,poll_time),
                                             eccb_stat.data64);
+
+            // Limited in callout: no PNOR target, so calling out processor
+            l_err->addHwCallout(
+                            TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                            HWAS::SRCI_PRIORITY_HIGH,
+                            HWAS::NO_DECONFIG,
+                            HWAS::GARD_NULL );
+
+            addFFDCRegisters(l_err);
             l_err->collectTrace(PNOR_COMP_NAME);
             l_err->collectTrace(XSCOM_COMP_NAME);
             //@todo (RTC:37744) - Any cleanup or recovery needed?
@@ -1451,7 +1821,7 @@ errlHndl_t PnorDD::writeLPC(uint32_t i_addr,
         if( l_err ) { break; }
 
         // check for errors
-        if( (eccb_stat.data64 & LPC_STAT_REG_ERROR_MASK)
+        if( (eccb_stat.data64 & ECCB_LPC_STAT_REG_ERROR_MASK)
             || (eccb_stat.op_done == 0) )
         {
             TRACFCOMP(g_trac_pnor, "PnorDD::writeLPC> Error or timeout from LPC Status Register : i_addr=0x%.8X, status=0x%.16X", i_addr, eccb_stat.data64 );
@@ -1470,6 +1840,14 @@ errlHndl_t PnorDD::writeLPC(uint32_t i_addr,
                                             PNOR::RC_LPC_ERROR,
                                             TWO_UINT32_TO_UINT64(0,i_addr),
                                             eccb_stat.data64);
+            // Limited in callout: no PNOR target, so calling out processor
+            l_err->addHwCallout(
+                            TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                            HWAS::SRCI_PRIORITY_HIGH,
+                            HWAS::NO_DECONFIG,
+                            HWAS::GARD_NULL );
+
+            addFFDCRegisters(l_err);
             l_err->collectTrace(PNOR_COMP_NAME);
             l_err->collectTrace(XSCOM_COMP_NAME);
             //@todo (RTC:37744) - Any cleanup or recovery needed?
@@ -1555,6 +1933,7 @@ errlHndl_t PnorDD::compareAndWriteBlock(uint32_t i_blockStart,
                 l_err = bufferedSfcRead(i_blockStart,
                                         i_writeStart-i_blockStart,
                                         read_data);
+                if( l_err ) { break; }
             }
 
             //Get data after write section
@@ -1568,9 +1947,7 @@ errlHndl_t PnorDD::compareAndWriteBlock(uint32_t i_blockStart,
                 l_err =  bufferedSfcRead(i_writeStart+i_bytesToWrite,
                                            tail_length,
                                            tail_buffer);
-                if( l_err )
-                {
-                    break; }
+                if( l_err ) { break; }
             }
 
             // erase the flash
@@ -1588,6 +1965,7 @@ errlHndl_t PnorDD::compareAndWriteBlock(uint32_t i_blockStart,
                 l_err = bufferedSfcWrite(i_blockStart,
                                          i_writeStart-i_blockStart,
                                          read_data);
+                if( l_err ) { break; }
             }
 
             //Write data after new data to write
@@ -1667,7 +2045,9 @@ errlHndl_t PnorDD::eraseFlash(uint32_t i_address)
                                             PNOR::MOD_PNORDD_ERASEFLASH,
                                             PNOR::RC_LPC_ERROR,
                                             TWO_UINT32_TO_UINT64(0,i_address),
-                                            findEraseBlock(i_address));
+                                            findEraseBlock(i_address),
+                                            true /*Add HB SW Callout*/ );
+            l_err->collectTrace(PNOR_COMP_NAME);
             break;
         }
 
@@ -1758,7 +2138,9 @@ errlHndl_t PnorDD::eraseFlash(uint32_t i_address)
                                             PNOR::MOD_PNORDD_ERASEFLASH,
                                             PNOR::RC_UNSUPPORTED_OPERATION,
                                             static_cast<uint64_t>(cv_nor_chipid),
-                                            i_address);
+                                            i_address,
+                                            true /*Add HB SW Callout*/ );
+            l_err->collectTrace(PNOR_COMP_NAME);
         }
     } while(0);
 
diff --git a/src/usr/pnor/pnordd.H b/src/usr/pnor/pnordd.H
index e61ae0630..8d3f53486 100644
--- a/src/usr/pnor/pnordd.H
+++ b/src/usr/pnor/pnordd.H
@@ -83,7 +83,7 @@ class PnorDD
     bool usingL3Cache( );
 
 
-        // Enumeration values must match those in debug framework.
+    // Enumeration values must match those in debug framework.
     enum PnorMode_t {
         MODEL_UNKNOWN,   /**< Invalid */
         MODEL_MEMCPY,    /**< No LPC logic, just do memcpy into cache area */
@@ -160,6 +160,7 @@ class PnorDD
         SFC_CMD_SPACE,    /**< Indicate accessing command reg */
         SFC_CMDBUF_SPACE, /**< Indicate accessing command buffer space */
         SFC_MMIO_SPACE,   /**< Indicate accessing MMIO based Direct Reads */
+        SFC_LPC_SPACE,    /**< Indicate LPC Slave Space  */
     };
 
     /**
@@ -182,6 +183,30 @@ class PnorDD
         SFC_REG_OADRNB  = 0x90, /**< OADRNB : Direct Access OBP Window Base Address */
         SFC_REG_OADRNS  = 0x94, /**< OADRNS : DIrect Access OPB Window Size */
         SFC_REG_CHIPIDCONF  = 0x9C, /**< CHIPIDCONF : config ChipId CMD */
+        SFC_REG_ERRCONF = 0x6C, /**< ERRCONF : Configures error counts that
+                                               cause interupts */
+        SFC_REG_ERRTAG  = 0x1C, /**< ERRTAG : Holds Control Info of Error */
+        SFC_REG_ERROFF  = 0x20, /**< ERROFF : Holds Address Info of Error */
+        SFC_REG_ERRSYN  = 0x24, /**< ERRSYN : Holds Syndrome That Caused Error*/
+        SFC_REG_ERRDATH = 0x28, /**< ERRDATH : Holds Most Signifcant Word of
+                                               Double Word That Caused Error */
+        SFC_REG_ERRDATL = 0x2C, /**< ERRDATL : Holds Least Signifcant Word of
+                                               Double Word That Caused Error */
+        SFC_REG_ERRCNT  = 0x30, /**< ERRCNT : Counts The Number Of Errors */
+        SFC_REG_CLRCNT  = 0x34, /**< CLRCNT : Which Bits To Clear In ERRCNT */
+        SFC_REG_ERRINJ  = 0x38, /**< ERRINJ : Force Errors Into Read Paths */
+        SFC_REG_PROTA   = 0x70, /**< PROTA : Write Protect Range Address Base */
+        SFC_REG_PROTM   = 0x74, /**< PROTM : Write Protect Range Size */
+        SFC_REG_ECCADDR = 0x78, /**< ECCADDR : ECC Disable Range Base Address */
+        SFC_REG_ECCRNG  = 0x7C, /**< ECCRNG : ECC Disable Range Size */
+        SFC_REG_ERRORS  = 0x00, /**< ERRORS : Collection of Error Status Bits */
+        SFC_REG_INTMSK  = 0x04, /**< INTMSK : Record of Events That Could Lead
+                                              To Interupt */
+        SFC_REG_INTENM  = 0x14, /**< INTENM : Controls Which Events Lead
+                                              To Interupts */
+        SFC_REG_CONF2   = 0x18, /**< CONF2 : SPI Configuration */
+        SFC_REG_CONF3   = 0x50, /**< CONF3 : SPI Recovery */
+
     };
 
     /**
@@ -201,6 +226,7 @@ class PnorDD
         SFC_OP_ABORT      = 0x6F, /**< Super-Abort */
         SFC_OP_START4BA   = 0x37, /**< Start 4BA */
         SFC_OP_END4BA     = 0x69, /**< End 4BA */
+        SFC_OP_INVALID    = 0x00, /**< Invalid - used for testing */
     };
 
 
@@ -215,6 +241,7 @@ class PnorDD
     };
 
 
+
     /**
      * Flags used to trigger Hardware workarounds
      */
@@ -245,15 +272,79 @@ class PnorDD
         uint32_t data32;
         struct
         {
-            uint32_t unused    : 30; /**< 0:29 = Not Currently Used  */
-            uint32_t timeout   : 1;  /**< 30   = Timeout */
-            uint32_t done      : 1;  /**< 31   = Done */
+            uint32_t unused     : 20; /**< 0:19 = Not Currently Used  */
+            uint32_t eccerrcntr : 1;  /**< 20 Threshold for SRAM ECC errors */
+            uint32_t eccues     : 1;  /**< 21 SRAM cmd uncorrectable ECC error*/
+            uint32_t unused_22  : 3;  /**< 22:24 = Not Currently Used */
+            uint32_t cmdexe     : 1;  /**< 25 Previous cmd is in progress */
+            uint32_t cmdwait    : 1;  /**< 26 Previous cmd waiting to execute */
+            uint32_t illegal    : 1;  /**< 27 Previous op illegal */
+            uint32_t eccerrcntn : 1;  /**< 28 Threshold for Flash ECC errors */
+            uint32_t eccuen     : 1;  /**< 29 Flash cmd uncorrectable ECC err */
+            uint32_t timeout    : 1;  /**< 30 Timeout */
+            uint32_t done       : 1;  /**< 31 Done */
         };
         SfcStatReg_t() : data32(0) {};
     };
 
 
     /**
+     * @brief LPC Slave Registers
+     *    These are offsets within the LPC Slave Register Space
+     */
+    enum LpcSlaveRegAddr {
+        LPC_SLAVE_REG_STATUS  = 0x14, /**< STATUS: read-only  */
+        LPC_SLAVE_REG_RESET   = 0x14, /**< RESET : write-only */
+    };
+
+    /**
+     * @brief  LPC Slave Status Register Layout
+     */
+    union LpcSlaveStatReg_t
+    {
+        uint32_t data32;
+        struct
+        {
+            uint32_t lbusowner       : 2;   /**< 0:1 = Local Bus Owner */
+            uint32_t lbusparityerror : 1;   /**< 2   = Local Bus Parity Error */
+            uint32_t lbus2opberr     : 3;   /**< 3:5 = Errors From LBUS2OPB */
+            uint32_t unused          : 26;  /**< 6:21 = Not Currently Used */
+        };
+        LpcSlaveStatReg_t() : data32(0) {};
+    };
+
+    /**
+     * @brief  LPC Slave Reset Register Layout
+     */
+    union LpcSlaveResetReg_t
+    {
+        uint32_t data32;
+        struct
+        {
+            uint32_t lpcslave     : 1;  /**< 0 Reset LPC Slave */
+            uint32_t lpcslaveerrs : 1;  /**< 1 Reset LPC Slave Errors */
+            uint32_t localbus     : 1;  /**< 2 Reset Local Bus */
+            uint32_t unused       : 29; /**< 4:31 = Not Currently Used */
+        };
+        LpcSlaveResetReg_t() : data32(0) {};
+    };
+
+
+    /**
+     * @brief LPC Slave LBUS2OPB Errors
+     *        Translation of LPC Slave Status Register Bits 3:5
+     */
+    enum LpcSlaveLbus2OpbErrors {
+        LBUS2OPB_ADDR_PARITY_ERR    = 0b010, /**< Address Parity Error */
+        LBUS2OPB_INVALID_SELECT_ERR = 0b001, /**< Invalid Select Error */
+        LBUS2OPB_DATA_PARITY_ERR    = 0b011, /**< Data Parity Error */
+        LBUS2OPB_MONITOR_ERR        = 0b100, /**< Monitor Error */
+        LBUS2OPB_TIMEOUT_ERR        = 0b101, /**< Timeout Error */
+    };
+
+
+
+    /**
      * @brief Write a SFC Register
      *
      * @parm i_range  SFC Address Range
@@ -406,9 +497,10 @@ class PnorDD
 
         // Default Values to set for all operations
         // 1101.0100.0000.000x.0000.0001.0000.0000.<address>
-        LPC_CTL_REG_DEFAULT = 0xD400010000000000,
+        ECCB_LPC_CTL_REG_DEFAULT = 0xD400010000000000,
 
-        LPC_STAT_REG_ERROR_MASK = 0xFC0000000007F700, /**< Error Bits */
+        // Error bits: 41-43, 56  (52=cmd complete) (not 57: only non-fw use)
+        ECCB_LPC_STAT_REG_ERROR_MASK = 0x0000000000700080, /**< Error Bits */
 
         ERASE_COUNT_MAX = 64,  /**<Max number of tracked erase blocks */
         ERASESIZE_BYTES_DEFAULT = 4 * KILOBYTE, /**< Min Erase Block (bytes) */
@@ -531,6 +623,21 @@ class PnorDD
                           size_t i_size );
 
     /**
+     * @brief Check For Errors in Status Registers
+     *
+     * @return Error log if error found
+     */
+    errlHndl_t checkForErrors( void );
+
+    /**
+     * @brief Add FFDC Error Registers to an existing Error Log
+     *
+     * @parm io_errl    Error Log To Add FFDC To
+     *
+     */
+    void addFFDCRegisters(errlHndl_t & io_errl);
+
+    /**
      * @brief  ECCB Control Register Layout
      */
     union ControlReg_t
@@ -548,7 +655,7 @@ class PnorDD
             uint64_t address     : 32; /**< 32:63 = LPC Address */
         };
 
-        ControlReg_t() : data64(LPC_CTL_REG_DEFAULT) {};
+        ControlReg_t() : data64(ECCB_LPC_CTL_REG_DEFAULT) {};
     };
 
     /**
@@ -559,18 +666,22 @@ class PnorDD
         uint64_t data64;
         struct
         {
-            uint64_t pib_errors  : 6;  /**< 0:5  */
+            uint64_t unused      : 6;  /**< 0:5  */
             uint64_t read_data   : 32; /**< 6:37 */
-            uint64_t unused1     : 6;  /**< 38:43 */
+            uint64_t unused1     : 3;  /**< 38:40 */
+            uint64_t eccb_err    : 3;  /**< 41:43 = ECCB_Error_Info */
             uint64_t busy        : 1;  /**< 44 = Operation Busy */
-            uint64_t errors1     : 7;  /**< 45:51 */
-            uint64_t op_done     : 1;  /**< 52 */
-            uint64_t errors2     : 3;  /**< 53:55 */
-            uint64_t unused2     : 8;  /**< 56:63 */
+            uint64_t unused2     : 7;  /**< 45:51 */
+            uint64_t op_done     : 1;  /**< 52 = Command Complete */
+            uint64_t unused3     : 3;  /**< 53:55 */
+            uint64_t addr_parity_err : 1; /**< 56 = ECC Address Register
+                                                    Parity Error */
+            uint64_t unused4     : 7;  /**< 57:63 */
         };
         StatusReg_t() : data64(0) {};
     };
 
+
   private: // Variables
 
     // NOTE: The layout of the variables in this class must be maintained
@@ -635,6 +746,12 @@ class PnorDD
      */
     uint64_t iv_fakeSize;
 
+    /**
+     * @brief Indicates if class is currently collecting FFDC data
+     *
+     */
+    bool iv_ffdc_active;
+
     // Needed for testcases
     friend class PnorDdTest;
 };
diff --git a/src/usr/pnor/pnorrp.C b/src/usr/pnor/pnorrp.C
index 575d568ec..86aaed140 100644
--- a/src/usr/pnor/pnorrp.C
+++ b/src/usr/pnor/pnorrp.C
@@ -141,7 +141,9 @@ void PnorRP::init( errlHndl_t   &io_rtaskRetErrl )
                                 PNOR::MOD_PNORRP_DIDSTARTUPFAIL,
                                 PNOR::RC_BAD_STARTUP_RC,
                                 rc,
-                                0   );
+                                0,
+                                true /*Add HB SW Callout*/ );
+
         l_errl->collectTrace(PNOR_COMP_NAME);
     }
 
@@ -261,8 +263,8 @@ void PnorRP::initDaemon()
 
     if( l_errhdl )
     {
-        errlCommit(l_errhdl,PNOR_COMP_ID);
         iv_startupRC = l_errhdl->reasonCode();
+        errlCommit(l_errhdl,PNOR_COMP_ID);
     }
 
     // call ErrlManager function - tell him that PNOR is ready!
@@ -1020,7 +1022,8 @@ errlHndl_t PnorRP::computeSection( uint64_t i_vaddr,
                                          PNOR::MOD_PNORRP_COMPUTESECTION,
                                          PNOR::RC_INVALID_ADDRESS,
                                          i_vaddr,
-                                         0);
+                                         0,
+                                         true /*Add HB SW Callout*/);
         errhdl->collectTrace(PNOR_COMP_NAME);
         return errhdl;
     }
diff --git a/src/usr/pnor/test/pnorddtest.H b/src/usr/pnor/test/pnorddtest.H
index 546240cb9..5f91b8f07 100644
--- a/src/usr/pnor/test/pnorddtest.H
+++ b/src/usr/pnor/test/pnorddtest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2013              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -972,6 +972,132 @@ class PnorDdTest : public CxxTest::TestSuite
     }
 
 
+    /**
+     * @brief PNOR DD Force Fails testcase
+     */
+/****************************************************************************/
+/* NOTE:  TEST DISABLED!!!  remove "_" before test name to re-enable        */
+/****************************************************************************/
+    void _test_forceFails(void)
+    {
+        errlHndl_t l_err = NULL;
+        uint64_t fails = 0;
+        uint64_t total = 0;
+        PnorDD* pnordd = new PnorDD();
+        uint32_t l_poll = 0;
+        PnorDD::SfcCmdReg_t sfc_cmd;
+
+        do{
+            TRACFCOMP(g_trac_pnor, "PnorDdTest::test_forceFails> starting - expect to see errors");
+
+            /*******************************/
+            /*  Send in an invalid OP Code */
+            /*******************************/
+            sfc_cmd.opcode = PnorDD::SFC_OP_INVALID;
+            sfc_cmd.length = 0;
+
+            mutex_lock(&(pnordd->cv_mutex));
+            l_err = pnordd->writeRegSfc(PnorDD::SFC_CMD_SPACE,
+                                        PnorDD::SFC_REG_CMD,
+                                        sfc_cmd.data32);
+            total++;
+            if (l_err)
+            {
+                TS_FAIL("PnorDdTest::test_forceFails> Get Chip Id cmd failed! Error committed.");
+                fails++;
+
+                // Unlock mutex for Error Log to be commited
+                mutex_unlock(&(pnordd->cv_mutex));
+                errlCommit(l_err,PNOR_COMP_ID);
+
+                // Lock mutex for next operation
+                mutex_lock(&(pnordd->cv_mutex));
+            }
+
+            // Poll for complete status without waiting
+            l_err = pnordd->pollSfcOpComplete();
+            mutex_unlock(&(pnordd->cv_mutex));
+
+            total++;
+            if ( l_err == NULL )
+            {
+                TS_FAIL("PnorDdTest::test_forceFails> Failed to create illegal opcode error!");
+                fails++;
+            }
+            else
+            {
+                // error correctly created - delete it
+                delete l_err;
+            }
+
+            /*******************************************************/
+            /* Issue a cmd but poll for completion without waiting */
+            /*******************************************************/
+            //Issue Get Chip ID command
+            sfc_cmd.opcode = PnorDD::SFC_OP_CHIPID;
+            sfc_cmd.length = 0;
+
+            mutex_lock(&(pnordd->cv_mutex));
+            l_err = pnordd->writeRegSfc(PnorDD::SFC_CMD_SPACE,
+                                        PnorDD::SFC_REG_CMD,
+                                        sfc_cmd.data32);
+
+            total++;
+            if (l_err)
+            {
+                TS_FAIL("PnorDdTest::test_forceFails> Get Chip Id cmd failed! Error committed.");
+                fails++;
+
+                // Unlock mutex for Error Log to be commited
+                mutex_unlock(&(pnordd->cv_mutex));
+                errlCommit(l_err,PNOR_COMP_ID);
+
+                // Lock mutex for next operation
+                mutex_lock(&(pnordd->cv_mutex));
+            }
+
+            // Poll for complete status without waiting
+            l_err = pnordd->pollSfcOpComplete(l_poll);
+
+            total++;
+            if ( l_err == NULL )
+            {
+                TS_FAIL("PnorDdTest::test_forceFails> pollSfcOpCompletel(l_poll=0) Failed to create an error!");
+                fails++;
+            }
+            else
+            {
+                // error correctly created - delete it
+                delete l_err;
+            }
+
+            /*******************************************************/
+            /* Cleanup: poll to make sure last operation completes */
+            /*         before continuing                           */
+            /*******************************************************/
+            l_err = pnordd->pollSfcOpComplete();
+            mutex_unlock(&(pnordd->cv_mutex));
+
+            total++;
+            if (l_err)
+            {
+                TS_FAIL("PnorDdTest::test_forceFails> Cleanup polling failed! Error committed.");
+                errlCommit(l_err,PNOR_COMP_ID);
+                fails++;
+            }
+
+
+        }while(0);
+
+        if( pnordd )
+        {
+            delete pnordd;
+        }
+
+        TRACFCOMP(g_trac_pnor, "PnorDdTest::test_forceFails> %d/%d fails", fails, total );
+
+    }
+
 };
 
 /*Not really a real test, just using to verify ext image is loading properly.