OCC Checkpoint handling improvements

Change-Id: Ibf8712fe560c022be939f73fcc78fb610a2bcabb
RTC: 153923
ForwardPort: yes
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/24622
Reviewed-by: Martha Broyles <mbroyles@us.ibm.com>
Reviewed-by: Sheldon R. Bailey <baileysh@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/24831
Tested-by: Jenkins Server
Tested-by: FSP CI Jenkins

5 files changed, 284 insertions, 176 deletions

diff --git a/src/include/usr/htmgt/htmgt_reasoncodes.H b/src/include/usr/htmgt/htmgt_reasoncodes.H
index d6a106949..6d523e752 100644
--- a/src/include/usr/htmgt/htmgt_reasoncodes.H
+++ b/src/include/usr/htmgt/htmgt_reasoncodes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2014,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2014,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -42,6 +42,7 @@ namespace HTMGT
         HTMGT_MOD_BUILD_OCCS            = 0x08,
         HTMGT_MOD_PASS_THRU             = 0x09,
         HTMGT_MOD_CFG_FREQ_POINTS       = 0x0A,
+        HTMGT_MOD_WAIT_FOR_CHECKPOINT   = 0x0B,
         HTMGT_MOD_OCC_POLL              = 0x76,
         HTMGT_MOD_BAD_FRU_CALLOUTS      = 0x7D,
         HTMGT_MOD_MISMATCHING_SEVERITY  = 0x7F,
diff --git a/src/usr/htmgt/htmgt.C b/src/usr/htmgt/htmgt.C
index e2e508025..502b661e0 100644
--- a/src/usr/htmgt/htmgt.C
+++ b/src/usr/htmgt/htmgt.C
@@ -55,7 +55,7 @@ namespace HTMGT
     void processOccStartStatus(const bool i_startCompleted,
                                TARGETING::Target * i_failedOccTarget)
     {
-        TMGT_INF(">>processOccStartStatus(%d,0x%p)",
+        TMGT_INF(">>processOccStartStatus(%d,%p)",
                  i_startCompleted, i_failedOccTarget);
         errlHndl_t l_err = NULL;
         uint32_t l_huid = 0;
@@ -88,7 +88,11 @@ namespace HTMGT
 #endif
 
                         // Make sure OCCs are ready for communication
-                        OccManager::waitForOccCheckpoint();
+                        l_err = OccManager::waitForOccCheckpoint();
+                        if (l_err)
+                        {
+                            break;
+                        }
 
 #ifdef __HOSTBOOT_RUNTIME
                         // TODO RTC 124738  Final solution TBD
@@ -222,7 +226,7 @@ namespace HTMGT
     // Notify HTMGT that an OCC has an error to report
     void processOccError(TARGETING::Target * i_procTarget)
     {
-        TMGT_INF(">>processOccError(0x%p)", i_procTarget);
+        TMGT_INF(">>processOccError(%p)", i_procTarget);
 
         TARGETING::Target* sys = NULL;
         TARGETING::targetService().getTopLevelTarget(sys);
@@ -298,7 +302,7 @@ namespace HTMGT
     // Notify HTMGT that an OCC has failed and needs to be reset
     void processOccReset(TARGETING::Target * i_proc)
     {
-        TMGT_INF(">>processOccReset(0x%p)", i_proc);
+        TMGT_INF(">>processOccReset(%p)", i_proc);
         errlHndl_t errl = NULL;
         TARGETING::Target * failedOccTarget = NULL;
 
@@ -314,12 +318,14 @@ namespace HTMGT
             return;
         }
 
-        // Get functional OCC (one per proc)
-        TARGETING::TargetHandleList pOccs;
-        getChildChiplets(pOccs, i_proc, TARGETING::TYPE_OCC);
-        if (pOccs.size() > 0)
+        if (i_proc)
         {
-            failedOccTarget = pOccs[0];
+            TARGETING::TargetHandleList pOccs;
+            getChildChiplets(pOccs, i_proc, TARGETING::TYPE_OCC);
+            if (pOccs.size() > 0)
+            {
+                failedOccTarget = pOccs[0];
+            }
         }
 
         if(NULL != failedOccTarget)
@@ -464,183 +470,190 @@ namespace HTMGT
         htmgtReasonCode failingSrc = HTMGT_RC_NO_ERROR;
         o_rspLength = 0;
 
-        if ((i_cmdLength > 0) && (NULL != i_cmdData))
+        err = OccManager::buildOccs();
+        if (NULL == err)
         {
-            switch (i_cmdData[0])
+            if ((i_cmdLength > 0) && (NULL != i_cmdData))
             {
-                case PASSTHRU_OCC_STATUS:
-                    TMGT_INF("passThruCommand: OCC Status");
-                    OccManager::getOccData(o_rspLength, o_rspData);
-                    break;
-
-                case PASSTHRU_GENERATE_MFG_PSTATE:
-                    if (i_cmdLength == 1)
-                    {
-                        TMGT_INF("passThruCommand: Generate MFG pstate tables",
-                                 i_cmdData[1]);
-                        err = genPstateTables(false);
-                    }
-                    else
-                    {
-                        TMGT_ERR("passThruCommand: invalid generate pstate "
-                                 "command length %d", i_cmdLength);
-                        /*@
-                         * @errortype
-                         * @reasoncode   HTMGT_RC_INVALID_LENGTH
-                         * @moduleid     HTMGT_MOD_PASS_THRU
-                         * @userdata1    command data[0-7]
-                         * @userdata2    command data length
-                         * @devdesc      Invalid pass thru command data length
-                         */
-                        failingSrc = HTMGT_RC_INVALID_LENGTH;
-                    }
-                    break;
+                switch (i_cmdData[0])
+                {
+                    case PASSTHRU_OCC_STATUS:
+                        TMGT_INF("passThruCommand: OCC Status");
+                        OccManager::getOccData(o_rspLength, o_rspData);
+                        break;
 
-                case PASSTHRU_LOAD_PSTATE:
-                    if (i_cmdLength == 2)
-                    {
-                        const uint8_t pstateType = i_cmdData[1];
-                        if ((0 == pstateType) || (1 == pstateType))
+                    case PASSTHRU_GENERATE_MFG_PSTATE:
+                        if (i_cmdLength == 1)
                         {
-                            TMGT_INF("passThruCommand: Load pstate tables "
-                                     "(type: %d)", pstateType);
-                            // 0 = Normal Pstate Tables
-                            err = OccManager::loadPstates(0 == pstateType);
+                            TMGT_INF("passThruCommand: Generate MFG pstate "
+                                     "tables", i_cmdData[1]);
+                            err = genPstateTables(false);
                         }
                         else
                         {
-                            TMGT_ERR("passThruCommand: invalid pstate type "
-                                     "specified: %d", pstateType);
+                            TMGT_ERR("passThruCommand: invalid generate pstate "
+                                     "command length %d", i_cmdLength);
                             /*@
                              * @errortype
-                             * @reasoncode   HTMGT_RC_INVALID_PARAMETER
+                             * @reasoncode   HTMGT_RC_INVALID_LENGTH
                              * @moduleid     HTMGT_MOD_PASS_THRU
                              * @userdata1    command data[0-7]
                              * @userdata2    command data length
-                             * @devdesc      Invalid load pstate table type
+                             * @devdesc      Invalid pass thru command data len
                              */
-                            failingSrc = HTMGT_RC_INVALID_PARAMETER;
+                            failingSrc = HTMGT_RC_INVALID_LENGTH;
                         }
-                    }
-                    else
-                    {
-                        TMGT_ERR("passThruCommand: invalid load pstate "
-                                 "command length %d", i_cmdLength);
-                        failingSrc = HTMGT_RC_INVALID_LENGTH;
-                    }
-                    break;
+                        break;
 
-                case PASSTHRU_SEND_OCC_COMMAND:
-                    if (i_cmdLength >= 3)
-                    {
-                        const uint8_t occInstance = i_cmdData[1];
-                        const occCommandType occCmd =
-                            (occCommandType)i_cmdData[2];
-                        const uint16_t dataLen = i_cmdLength-3;
-                        Occ *occPtr = OccManager::getOcc(occInstance);
-                        if (occPtr)
+                    case PASSTHRU_LOAD_PSTATE:
+                        if (i_cmdLength == 2)
                         {
-                            TMGT_INF("passThruCommand: Send OCC%d command "
-                                     "0x%02X (%d bytes)",
-                                     occInstance, occCmd, dataLen);
-                            OccCmd cmd(occPtr, occCmd, dataLen, &i_cmdData[3]);
-                            err = cmd.sendOccCmd();
-                            if (err != NULL)
+                            const uint8_t pstateType = i_cmdData[1];
+                            if ((0 == pstateType) || (1 == pstateType))
                             {
-                                TMGT_ERR("passThruCommand: OCC%d cmd 0x%02X "
-                                         "failed with rc 0x%04X",
-                                         occInstance, occCmd,
-                                         err->reasonCode());
+                                TMGT_INF("passThruCommand: Load pstate tables "
+                                         "(type: %d)", pstateType);
+                                // 0 = Normal Pstate Tables
+                                err = OccManager::loadPstates(0 == pstateType);
                             }
                             else
                             {
-                                uint8_t *rspPtr = NULL;
-                                o_rspLength = cmd.getResponseData(rspPtr);
-                                memcpy(o_rspData, rspPtr, o_rspLength);
-                                TMGT_INF("passThruCommand: OCC%d rsp status "
-                                         "0x%02X (%d bytes)", occInstance,
-                                         o_rspData[2], o_rspLength);
+                                TMGT_ERR("passThruCommand: invalid pstate "
+                                         "type specified: %d", pstateType);
+                                /*@
+                                 * @errortype
+                                 * @reasoncode   HTMGT_RC_INVALID_PARAMETER
+                                 * @moduleid     HTMGT_MOD_PASS_THRU
+                                 * @userdata1    command data[0-7]
+                                 * @userdata2    command data length
+                                 * @devdesc      Invalid load pstate table type
+                                 */
+                                failingSrc = HTMGT_RC_INVALID_PARAMETER;
                             }
                         }
                         else
                         {
-                            TMGT_ERR("passThruCommand: Unable to find OCC%d",
-                                     occInstance);
-                            /*@
-                             * @errortype
-                             * @reasoncode   HTMGT_RC_OCC_UNAVAILABLE
-                             * @moduleid     HTMGT_MOD_PASS_THRU
-                             * @userdata1    command data[0-7]
-                             * @userdata2    command data length
-                             * @devdesc      Specified OCC not available
-                             */
-                            failingSrc = HTMGT_RC_OCC_UNAVAILABLE;
+                            TMGT_ERR("passThruCommand: invalid load pstate "
+                                     "command length %d", i_cmdLength);
+                            failingSrc = HTMGT_RC_INVALID_LENGTH;
                         }
-                    }
-                    else
-                    {
-                        TMGT_ERR("passThruCommand: invalid OCC command "
-                                 "length %d", i_cmdLength);
-                        failingSrc = HTMGT_RC_INVALID_LENGTH;
-                    }
-                    break;
-
-                case PASSTHRU_CLEAR_RESET_COUNTS:
-                    TMGT_INF("passThruCommand: Clear all OCC reset counts");
-                    OccManager::clearResetCounts();
-                    break;
+                        break;
 
-                case PASSTHRU_EXIT_SAFE_MODE:
-                    {
-                        TMGT_INF("passThruCommand: Clear Safe Mode");
-                        // Clear OCC reset counts and failed flags
-                        OccManager::clearResetCounts();
-                        // Clear safe mode reason
-                        OccManager::updateSafeModeReason(0, 0);
-                        // Clear system safe mode flag/attribute
-                        TARGETING::Target* sys = NULL;
-                        TARGETING::targetService().getTopLevelTarget(sys);
-                        const uint8_t safeMode = 0;
-                        if(sys)
+                    case PASSTHRU_SEND_OCC_COMMAND:
+                        if (i_cmdLength >= 3)
                         {
-                            sys->setAttr<TARGETING::ATTR_HTMGT_SAFEMODE>
-                                (safeMode);
+                            const uint8_t occInstance = i_cmdData[1];
+                            const occCommandType occCmd =
+                                (occCommandType)i_cmdData[2];
+                            const uint16_t dataLen = i_cmdLength-3;
+                            Occ *occPtr = OccManager::getOcc(occInstance);
+                            if (occPtr)
+                            {
+                                TMGT_INF("passThruCommand: Send OCC%d command "
+                                         "0x%02X (%d bytes)",
+                                         occInstance, occCmd, dataLen);
+                                OccCmd cmd(occPtr, occCmd, dataLen,
+                                           &i_cmdData[3]);
+                                err = cmd.sendOccCmd();
+                                if (err != NULL)
+                                {
+                                    TMGT_ERR("passThruCommand: OCC%d cmd "
+                                             "0x%02X failed with rc 0x%04X",
+                                             occInstance, occCmd,
+                                             err->reasonCode());
+                                }
+                                else
+                                {
+                                    uint8_t *rspPtr = NULL;
+                                    o_rspLength = cmd.getResponseData(rspPtr);
+                                    memcpy(o_rspData, rspPtr, o_rspLength);
+                                    TMGT_INF("passThruCommand: OCC%d rsp "
+                                             "status 0x%02X (%d bytes)",
+                                             occInstance, o_rspData[2],
+                                             o_rspLength);
+                                }
+                            }
+                            else
+                            {
+                                TMGT_ERR("passThruCommand: Unable to find "
+                                         "OCC%d", occInstance);
+                                /*@
+                                 * @errortype
+                                 * @reasoncode   HTMGT_RC_OCC_UNAVAILABLE
+                                 * @moduleid     HTMGT_MOD_PASS_THRU
+                                 * @userdata1    command data[0-7]
+                                 * @userdata2    command data length
+                                 * @devdesc      Specified OCC not available
+                                 */
+                                failingSrc = HTMGT_RC_OCC_UNAVAILABLE;
+                            }
                         }
-                        // Reset the OCCs (do not increment reset count
-                        // or attempt comm with OCC since they are in reset)
-                        TMGT_INF("passThruCommand: Calling resetOccs");
-                        err = OccManager::resetOccs(NULL, true, true);
-                        if (err != NULL)
+                        else
                         {
-                            TMGT_ERR("passThruCommand: resetOccs failed "
-                                     "with rc 0x%04X", err->reasonCode());
+                            TMGT_ERR("passThruCommand: invalid OCC command "
+                                     "length %d", i_cmdLength);
+                            failingSrc = HTMGT_RC_INVALID_LENGTH;
                         }
-                    }
-                    break;
+                        break;
 
-                default:
-                    TMGT_ERR("passThruCommand: Invalid command 0x%08X "
-                             "(%d bytes)", UINT32_GET(i_cmdData), i_cmdLength);
-                    /*@
-                     * @errortype
-                     * @reasoncode   HTMGT_RC_INVALID_DATA
-                     * @moduleid     HTMGT_MOD_PASS_THRU
-                     * @userdata1    command data[0-7]
-                     * @userdata2    command data length
-                     * @devdesc      Invalid pass thru command
-                     */
-                    failingSrc = HTMGT_RC_INVALID_DATA;
-                    break;
-            }
+                    case PASSTHRU_CLEAR_RESET_COUNTS:
+                        TMGT_INF("passThruCommand: Clear all OCC reset counts");
+                        OccManager::clearResetCounts();
+                        break;
 
-            if ((HTMGT_RC_NO_ERROR != failingSrc) && (NULL == err))
-            {
-                bldErrLog(err, HTMGT_MOD_PASS_THRU,
-                          failingSrc,
-                          UINT32_GET(i_cmdData),
-                          UINT32_GET(&i_cmdData[4]),
-                          0, i_cmdLength,
-                          ERRORLOG::ERRL_SEV_INFORMATIONAL);
+                    case PASSTHRU_EXIT_SAFE_MODE:
+                        {
+                            TMGT_INF("passThruCommand: Clear Safe Mode");
+                            // Clear OCC reset counts and failed flags
+                            OccManager::clearResetCounts();
+                            // Clear safe mode reason
+                            OccManager::updateSafeModeReason(0, 0);
+                            // Clear system safe mode flag/attribute
+                            TARGETING::Target* sys = NULL;
+                            TARGETING::targetService().getTopLevelTarget(sys);
+                            const uint8_t safeMode = 0;
+                            if(sys)
+                            {
+                                sys->setAttr<TARGETING::ATTR_HTMGT_SAFEMODE>
+                                    (safeMode);
+                            }
+                            // Reset the OCCs (do not increment reset count
+                            // or attempt comm with OCC since they are in reset)
+                            TMGT_INF("passThruCommand: Calling resetOccs");
+                            err = OccManager::resetOccs(NULL, true, true);
+                            if (err != NULL)
+                            {
+                                TMGT_ERR("passThruCommand: resetOccs failed "
+                                         "with rc 0x%04X", err->reasonCode());
+                            }
+                        }
+                        break;
+
+                    default:
+                        TMGT_ERR("passThruCommand: Invalid command 0x%08X "
+                                 "(%d bytes)",
+                                 UINT32_GET(i_cmdData), i_cmdLength);
+                        /*@
+                         * @errortype
+                         * @reasoncode   HTMGT_RC_INVALID_DATA
+                         * @moduleid     HTMGT_MOD_PASS_THRU
+                         * @userdata1    command data[0-7]
+                         * @userdata2    command data length
+                         * @devdesc      Invalid pass thru command
+                         */
+                        failingSrc = HTMGT_RC_INVALID_DATA;
+                        break;
+                }
+
+                if ((HTMGT_RC_NO_ERROR != failingSrc) && (NULL == err))
+                {
+                    bldErrLog(err, HTMGT_MOD_PASS_THRU,
+                              failingSrc,
+                              UINT32_GET(i_cmdData),
+                              UINT32_GET(&i_cmdData[4]),
+                              0, i_cmdLength,
+                              ERRORLOG::ERRL_SEV_INFORMATIONAL);
+                }
             }
         }
 
@@ -648,6 +661,5 @@ namespace HTMGT
 
     } // end passThruCommand()
 
-
 }
 
diff --git a/src/usr/htmgt/htmgt_occ.C b/src/usr/htmgt/htmgt_occ.C
index e0aa26c55..8b34c525a 100644
--- a/src/usr/htmgt/htmgt_occ.C
+++ b/src/usr/htmgt/htmgt_occ.C
@@ -39,9 +39,9 @@
 #include <sys/time.h>
 #include <ecmdDataBufferBase.H>
 #include <hwpf/hwp/occ/occAccess.H>
-
 #include <hwpf/hwp/occ/occ.H>
 #include <hwpf/hwp/occ/occ_common.H>
+#include <errl/errludlogregister.H>
 
 namespace HTMGT
 {
@@ -241,6 +241,37 @@ namespace HTMGT
     }
 
 
+    // Add channel 1 (circular buffer) SCOM data to elog
+    void Occ::collectCheckpointScomData(errlHndl_t i_err)
+    {
+        if (i_err)
+        {
+            TARGETING::ConstTargetHandle_t procTarget =
+                TARGETING::getParentChip(iv_target);
+            ERRORLOG::ErrlUserDetailsLogRegister l_scom_data(procTarget);
+            // Grab circular buffer scom data: (channel 1)
+            //0006B031  OCBCSR1 (Control/Status [1]  Register)
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6B031));
+            //0006A211  OCBSLCS1
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A211));
+            //0006A214  OCBSHCS1
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A214));
+            //0006A216  OCBSES1 (Indicates error that occur in an indirect ch)
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A216));
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A210));
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A213));
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6A217));
+            l_scom_data.addData(DEVICE_SCOM_ADDRESS(0x6B034));
+            l_scom_data.addToLog(i_err);
+        }
+        else
+        {
+            TMGT_ERR("collectCheckpointScomData: No error "
+                     "handle supplied for OCC%d", iv_instance);
+        }
+    } // end Occ::collectCheckpointScomData()
+
+
 
     /////////////////////////////////////////////////////////////////
 
@@ -380,13 +411,17 @@ namespace HTMGT
                          * @errortype
                          * @moduleid HTMGT_MOD_BUILD_OCCS
                          * @reasoncode HTMGT_RC_OCC_CRIT_FAILURE
+                         * @userdata1  OCC Instance
+                         * @userdata2  homer virtual address
                          * @devdesc Homer pointer is NULL, unable to communicate
                          *          with the OCCs.  Leaving system in safe mode.
                          */
                         bldErrLog(err,
                                   HTMGT_MOD_BUILD_OCCS,
                                   HTMGT_RC_OCC_CRIT_FAILURE,
-                                  0, 0, 0, 0,
+                                  0, instance,
+                                  (uint64_t)homer>>32,
+                                  (uint64_t)homer&0xFFFFFFFF,
                                   ERRORLOG::ERRL_SEV_UNRECOVERABLE);
                     }
                 }
@@ -871,12 +906,13 @@ namespace HTMGT
 
 
     // Wait for all OCCs to reach communications checkpoint
-    void OccManager::_waitForOccCheckpoint()
+    errlHndl_t OccManager::_waitForOccCheckpoint()
     {
+        errlHndl_t checkpointElog = NULL;
 #ifdef CONFIG_HTMGT
-        // Wait up to 10 seconds for all OCCs to be ready (100 * 100ms = 10s)
+        // Wait up to 15 seconds for all OCCs to be ready (150 * 100ms = 15s)
         const size_t NS_BETWEEN_READ = 100 * NS_PER_MSEC;
-        const size_t READ_RETRY_LIMIT = 100;
+        const size_t READ_RETRY_LIMIT = 150;
 
         if (iv_occArray.size() > 0)
         {
@@ -888,9 +924,12 @@ namespace HTMGT
                  pOcc++)
             {
                 bool occReady = false;
+                uint16_t lastCheckpoint = 0x0000;
 
                 while ((!occReady) && (retryCount++ < READ_RETRY_LIMIT))
                 {
+                    nanosleep(0, NS_BETWEEN_READ);
+
                     // Read SRAM response buffer to check for OCC checkpoint
                      errlHndl_t l_err = NULL;
                     const uint16_t l_length = 8;
@@ -900,11 +939,18 @@ namespace HTMGT
                                             l_buffer);
                     if (NULL == l_err)
                     {
-                        // Check response status for checkpoint
-                        if ((0x0E == l_buffer.getByte(6)) &&
-                            (0xFF == l_buffer.getByte(7)))
+                        // Check response status for checkpoint (byte 6-7)
+                        const uint16_t checkpoint = l_buffer.getHalfWord(3);
+                        if (checkpoint != lastCheckpoint)
                         {
-                            TMGT_INF("waitForOccCheckpoint OCC%d ready!",
+                            TMGT_INF("_waitForOccCheckpoint: OCC%d Checkpoint "
+                                     "0x%04X",
+                                     (*pOcc)->getInstance(), checkpoint);
+                            lastCheckpoint = checkpoint;
+                        }
+                        if (0x0EFF == checkpoint)
+                        {
+                            TMGT_INF("_waitForOccCheckpoint OCC%d ready!",
                                      (*pOcc)->getInstance());
 
                             occReady = true;
@@ -916,7 +962,7 @@ namespace HTMGT
                         if (false == throttleErrors)
                         {
                             throttleErrors = true;
-                            TMGT_ERR("waitForOccCheckpoint: error trying to "
+                            TMGT_ERR("_waitForOccCheckpoint: error trying to "
                                      "read OCC%d SRAM (rc=0x%04X)",
                                      (*pOcc)->getInstance(),
                                      l_err->reasonCode());
@@ -928,18 +974,47 @@ namespace HTMGT
                             l_err = NULL;
                         }
                     }
-
-                    nanosleep(0, NS_BETWEEN_READ);
                 }
 
                 if (!occReady)
                 {
-                    TMGT_ERR("waitForOccCheckpoint OCC%d still NOT ready!",
-                             (*pOcc)->getInstance());
+                    TMGT_CONSOLE("Final OCC%d Checkpoint NOT reached (0x%04X)",
+                                 (*pOcc)->getInstance(), lastCheckpoint);
+                    TMGT_ERR("_waitForOccCheckpoint OCC%d still NOT ready! "
+                             "(last checkpoint=0x%04X)",
+                             (*pOcc)->getInstance(), lastCheckpoint);
+                    errlHndl_t l_err = NULL;
+                    /*@
+                     * @errortype
+                     * @moduleid HTMGT_MOD_WAIT_FOR_CHECKPOINT
+                     * @reasoncode HTMGT_RC_OCC_NOT_READY
+                     * @userdata1 OCC instance
+                     * @userdata2 last OCC checkpoint
+                     * @devdesc Set of OCC state failed
+                     */
+                    bldErrLog(l_err, HTMGT_MOD_WAIT_FOR_CHECKPOINT,
+                              HTMGT_RC_OCC_NOT_READY,
+                              0, (*pOcc)->getInstance(), 0, lastCheckpoint,
+                              ERRORLOG::ERRL_SEV_PREDICTIVE);
+
+                    (*pOcc)->collectCheckpointScomData(l_err);
+                    if (NULL == checkpointElog)
+                    {
+                        // return the first elog
+                        checkpointElog = l_err;
+                        l_err = NULL;
+                    }
+                    else
+                    {
+                        ERRORLOG::errlCommit(l_err, HTMGT_COMP_ID);
+                    }
+
+                    (*pOcc)->failed(true);
                 }
             }
         }
 #endif
+        return checkpointElog;
     }
 
 
@@ -1217,9 +1292,9 @@ namespace HTMGT
     }
 
 
-    void OccManager::waitForOccCheckpoint()
+    errlHndl_t OccManager::waitForOccCheckpoint()
     {
-        Singleton<OccManager>::instance()._waitForOccCheckpoint();
+        return Singleton<OccManager>::instance()._waitForOccCheckpoint();
     }
 
     void OccManager::updateSafeModeReason(uint32_t i_src,
diff --git a/src/usr/htmgt/htmgt_occ.H b/src/usr/htmgt/htmgt_occ.H
index e29565fe6..eaef25059 100644
--- a/src/usr/htmgt/htmgt_occ.H
+++ b/src/usr/htmgt/htmgt_occ.H
@@ -266,6 +266,15 @@ namespace HTMGT
              */
             void updateOccPresentBits(uint8_t i_slavePresent);
 
+
+            /**
+             * @brief Add channel 1 (circular buffer) SCOM data to elog
+             *
+             * @param[in,out]  i_err  Error log to add data to
+             */
+            void collectCheckpointScomData(errlHndl_t i_err);
+
+
         private:   // functions
 
             /**
@@ -477,8 +486,13 @@ namespace HTMGT
              *        communicate and start handling commands.  This
              *        function will wait up to 10 seconds for all OCCs
              *        before returning to the caller.
+             *
+             * @return NULL on success, or error handle for the first OCC
+             *         that did not reach its checkpoint.  If additional OCCs
+             *         do not reach their checkpoint an elog will be committed
+             *         for each.
              */
-            static void waitForOccCheckpoint();
+            static errlHndl_t waitForOccCheckpoint();
 
 
             /**
@@ -668,7 +682,8 @@ namespace HTMGT
             /* See setOccState() above */
             errlHndl_t _setOccState(const occStateId i_state);
 
-            void _waitForOccCheckpoint();
+            /* See waitForOccCheckpoint() above */
+            errlHndl_t _waitForOccCheckpoint();
 
             /* See resetOccs() above */
             errlHndl_t _resetOccs(TARGETING::Target * i_failedOccTarget,
diff --git a/src/usr/htmgt/htmgt_occcmd.C b/src/usr/htmgt/htmgt_occcmd.C
index d842e8265..934cf0ec1 100644
--- a/src/usr/htmgt/htmgt_occcmd.C
+++ b/src/usr/htmgt/htmgt_occcmd.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2014,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2014,2016                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -868,6 +868,10 @@ namespace HTMGT
                                                (uint32_t)MAX_FFDC),
                                       1,  // version
                                       exceptionType); // subsection
+                    if (0xE1 == exceptionType)
+                    {
+                        iv_Occ->collectCheckpointScomData(l_excErr);
+                    }
                     ERRORLOG::errlCommit(l_excErr, HTMGT_COMP_ID);
 
                     // Save exception so we don't log it again
@@ -918,7 +922,8 @@ namespace HTMGT
         {
             TMGT_ERR("writeOccCmd: Error writing to OCC Circular Buffer,"
                      " rc=0x%04X", l_err->reasonCode());
-
+            iv_Occ->collectCheckpointScomData(l_err);
+            l_err->collectTrace("HTMGT");
             ERRORLOG::errlCommit(l_err, HTMGT_COMP_ID);
         }
 #endif