Checkpoint improvements

Change-Id: I08d76ac1db6e425a9690864f693f36cb848cead5
RTC: 153965
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/28188
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William A. Bryan <wilbryan@us.ibm.com>
Reviewed-by: Wael El-Essawy <welessa@us.ibm.com>

9 files changed, 124 insertions, 50 deletions

diff --git a/src/occ_405/amec/amec_init.c b/src/occ_405/amec/amec_init.c
index c4c16a6..ba40fcc 100644
--- a/src/occ_405/amec/amec_init.c
+++ b/src/occ_405/amec/amec_init.c
@@ -424,7 +424,7 @@ void amec_slave_init()
         rc2                                 //userdata2
     );
 
-    REQUEST_RESET(l_err);
+    CHECKPOINT_FAIL_AND_HALT(l_err);
   }
   else
   {
diff --git a/src/occ_405/cmdh/cmdh_fsp.c b/src/occ_405/cmdh/cmdh_fsp.c
index 51991c4..6e3e0c0 100755
--- a/src/occ_405/cmdh/cmdh_fsp.c
+++ b/src/occ_405/cmdh/cmdh_fsp.c
@@ -45,6 +45,10 @@ const int      OCB_CHANNEL_FSP_LINEAR          = 0;
 const int      OCB_CHANNEL_FSP_DOORBELL        = 1;
 const int      ALLOW_UNTRUSTED_ACCESS          = 1;
 
+// L_startup_trace flag will enable additional checkpoint data prior to
+// first communication with TMGT to help debug communication issues.
+static bool L_startup_trace = TRUE;
+
 // Do not change this without changing size of FSP Command Buffer
 // (and vice versa).  The Linear window requires an alignment on
 // a power of 2 boundry, and linear window registers require this
@@ -131,6 +135,22 @@ void clearCmdhWakeupCondition(eCmdhWakeupThreadMask i_cond)
 void notifyFspDoorbellReceived(void * i_arg)
 {
     notifyCmdhWakeupCondition(CMDH_WAKEUP_FSP_COMMAND);
+
+    if (L_startup_trace)
+    {
+        // Read/trace push queue register
+        const uint32_t l_data = in32(OCB_OCBSHCS1);
+        TRAC_INFO("notifyFspDoorbellReceived: OCB_OCBSHCS1=0x%08X", l_data);
+        // write data after checkpoint and update checkpoint length
+        G_fsp_msg.rsp->fields.data[3] = (l_data >> 24);
+        G_fsp_msg.rsp->fields.data[4] = (l_data >> 16) & 0xFF;
+        G_fsp_msg.rsp->fields.data[5] = (l_data >>  8) & 0xFF;
+        G_fsp_msg.rsp->fields.data[6] = (l_data      ) & 0xFF;
+        G_fsp_msg.rsp->fields.data[7] = 0x22;
+        G_fsp_msg.rsp->fields.data_length[1] = 8; // 8 bytes vs 3
+        dcache_flush_line((void *)CMDH_OCC_RESPONSE_BASE_ADDRESS);
+    }
+
 }
 
 
@@ -155,6 +175,21 @@ int cmdh_thread_wait_for_wakeup(void)
         ocb_request_schedule(&G_fsp_doorbell_ocb_request);
     }
 
+    if (L_startup_trace)
+    {
+        // Read/trace push queue register
+        const uint32_t l_data = in32(OCB_OCBSHCS1);
+        TRAC_INFO("cmdh_thread_wait_for_wakeup: OCB_OCBSHCS1=0x%08X", l_data);
+        // write data after checkpoint and update checkpoint length
+        G_fsp_msg.rsp->fields.data[3] = (l_data >> 24);
+        G_fsp_msg.rsp->fields.data[4] = (l_data >> 16) & 0xFF;
+        G_fsp_msg.rsp->fields.data[5] = (l_data >>  8) & 0xFF;
+        G_fsp_msg.rsp->fields.data[6] = (l_data      ) & 0xFF;
+        G_fsp_msg.rsp->fields.data[7] = 0x33;
+        G_fsp_msg.rsp->fields.data_length[1] = 8; // 8 bytes vs 3
+        dcache_flush_line((void *)CMDH_OCC_RESPONSE_BASE_ADDRESS);
+    }
+
     // Wait for someone to wakeup this thread
     l_rc = ssx_semaphore_pend(&G_cmdh_fsp_wakeup_thread, SSX_WAIT_FOREVER);
 
@@ -163,12 +198,12 @@ int cmdh_thread_wait_for_wakeup(void)
 
 // Function Specification
 //
-// Name: cmdh_fsp_init
+// Name: cmdh_comm_init
 //
-// Description: TODO -- Add description
+// Description: Setup doorbell for TMGT/HTMGT/BMC communication
 //
 // End Function Specification
-errlHndl_t cmdh_fsp_init(void)
+void cmdh_comm_init(void)
 {
     errlHndl_t            l_errlHndl = NULL;
 
@@ -205,8 +240,8 @@ errlHndl_t cmdh_fsp_init(void)
        G_occ_interrupt_type != FSP_SUPPORTED_OCC)
     {
         // Invalid interrupt type
-        CMDH_TRAC_ERR("cmdh_fsp_init: Invalid OCC interrupt type was detected! interrupt_type[%d]",
-                 G_occ_interrupt_type);
+        CMDH_TRAC_ERR("cmdh_comm_init: Invalid OCC interrupt type was detected! interrupt_type[%d]",
+                      G_occ_interrupt_type);
 
         /* @
          * @errortype
@@ -218,18 +253,17 @@ errlHndl_t cmdh_fsp_init(void)
          * @devdesc     Invalid OCC interrupt type was detected
          */
         l_errlHndl = createErrl(
-                CMDH_OCC_INTERRUPT_TYPE,           // modId
-                EXTERNAL_INTERFACE_FAILURE,        // reasoncode
-                OCC_NO_EXTENDED_RC,                // Extended reason code
-                ERRL_SEV_PREDICTIVE,               // Severity
-                NULL,                              // Trace Buf
-                DEFAULT_TRACE_SIZE,                // Trace Size
-                G_occ_interrupt_type,              // userdata1
-                0                                  // userdata2
-                );
+                                CMDH_OCC_INTERRUPT_TYPE,           // modId
+                                EXTERNAL_INTERFACE_FAILURE,        // reasoncode
+                                OCC_NO_EXTENDED_RC,                // Extended reason code
+                                ERRL_SEV_PREDICTIVE,               // Severity
+                                NULL,                              // Trace Buf
+                                DEFAULT_TRACE_SIZE,                // Trace Size
+                                G_occ_interrupt_type,              // userdata1
+                                0                                  // userdata2
+                               );
+        CHECKPOINT_FAIL_AND_HALT(l_errlHndl);
     }
-
-    return l_errlHndl;
 }
 
 // Function Specification
@@ -461,6 +495,15 @@ errlHndl_t cmdh_fsp_cmd_hndler(void)
             break;
         }
 
+        bool l_trace_cmd = FALSE;
+        if (L_startup_trace)
+        {
+            // Disable startup traces since the response buffer will no longer contain the checkpoint
+            L_startup_trace = FALSE;
+            // Trace the current command and response status
+            l_trace_cmd = TRUE;
+        }
+
         // Determine which buffer to read based on the sender ID
         if(l_sender_id == ATTN_SENDER_ID_HTMGT)
         {
@@ -590,6 +633,11 @@ errlHndl_t cmdh_fsp_cmd_hndler(void)
                 l_cmd_len = CMDH_FSP_CMD_SIZE;
             }
 
+            if (l_trace_cmd)
+            {
+                TRAC_INFO("cmdh_fsp_cmd_hndler: HTMGT command(cmd: 0x%02X, seq: 0x%02X)", G_htmgt_cmd_buffer.fields.cmd_type, G_htmgt_cmd_buffer.fields.seq);
+            }
+
             // Verify the command checksum
             l_cksm = checksum16(&G_htmgt_cmd_buffer.byte[0], l_cmd_len);
             if(l_cksm != CONVERT_UINT8_ARRAY_UINT16(G_htmgt_cmd_buffer.byte[l_cmd_len],
@@ -627,6 +675,10 @@ errlHndl_t cmdh_fsp_cmd_hndler(void)
             G_htmgt_rsp_buffer.byte[l_cmd_len]   = CONVERT_UINT16_UINT8_HIGH(l_cksm);
             G_htmgt_rsp_buffer.byte[l_cmd_len+1] = CONVERT_UINT16_UINT8_LOW(l_cksm);
             G_htmgt_rsp_buffer.fields.rc         = G_rsp_status;
+            if (l_trace_cmd)
+            {
+                TRAC_INFO("cmdh_fsp_cmd_hndler: HTMGT response(status: 0x%02X, dataLen: 0x%04X)", G_htmgt_rsp_buffer.fields.rc, l_data_len);
+            }
 
             // Need to write the final response to HOMER. Set up a copy request
             l_ssxrc = bce_request_create(&pba_copy,                           // block copy object
@@ -703,6 +755,11 @@ errlHndl_t cmdh_fsp_cmd_hndler(void)
                 l_cmd_len = CMDH_FSP_CMD_SIZE;
             }
 
+            if (l_trace_cmd)
+            {
+                TRAC_INFO("cmdh_fsp_cmd_hndler: TMGT command(cmd: 0x%02X, seq: 0x%02X)" , G_fsp_msg.cmd->fields.cmd_type, G_fsp_msg.cmd->fields.seq);
+            }
+
             // Verify Command Checksum
             l_cksm = checksum16(&G_fsp_msg.cmd->byte[0],(l_cmd_len));
             if(l_cksm != CONVERT_UINT8_ARRAY_UINT16(G_fsp_msg.cmd->byte[l_cmd_len],
@@ -743,6 +800,11 @@ errlHndl_t cmdh_fsp_cmd_hndler(void)
             G_fsp_msg.rsp->byte[l_cmd_len] = CONVERT_UINT16_UINT8_HIGH(l_cksm);
             G_fsp_msg.rsp->byte[l_cmd_len+1] = CONVERT_UINT16_UINT8_LOW(l_cksm);
 
+            if (l_trace_cmd)
+            {
+                TRAC_INFO("cmdh_fsp_cmd_hndler: TMGT response(status: 0x%02X, dataLen: 0x%04X)", G_rsp_status, l_data_len);
+            }
+
             // Copy the return status last (to indicate command completion)
             G_fsp_msg.rsp->fields.rc = G_rsp_status;
         }
diff --git a/src/occ_405/cmdh/cmdh_fsp.h b/src/occ_405/cmdh/cmdh_fsp.h
index 49d7fc9..14da318 100755
--- a/src/occ_405/cmdh/cmdh_fsp.h
+++ b/src/occ_405/cmdh/cmdh_fsp.h
@@ -151,6 +151,8 @@ typedef enum
     ERRL_RC_OCB_TIMER           = 0xE3,
     // Halting due to unmasked OCCLFIR bit being set (see OISR0 bit 2)
     ERRL_RC_OCC_HW_ERROR        = 0xE4,
+    // Halting due to failure during init
+    ERRL_RC_OCC_INIT_FAILURE    = 0xE5,
     // The command is being processed by OCC
     ERRL_RC_CMD_IN_PROGRESS     = 0xFF,
 } ERRL_RC;
@@ -316,7 +318,7 @@ int cmdh_thread_wait_for_wakeup(void);
 
 errlHndl_t cmdh_fsp_cmd_hndler(void);
 
-errlHndl_t cmdh_fsp_init(void);
+void cmdh_comm_init(void);
 
 void cmdh_build_errl_rsp(const cmdh_fsp_cmd_t * i_cmd_ptr,
                          cmdh_fsp_rsp_t       * o_rsp_ptr,
diff --git a/src/occ_405/cmdh/cmdh_thread.c b/src/occ_405/cmdh/cmdh_thread.c
index 1a5c08a..4dd66f9 100644
--- a/src/occ_405/cmdh/cmdh_thread.c
+++ b/src/occ_405/cmdh/cmdh_thread.c
@@ -57,18 +57,20 @@ void Cmd_Hndl_thread_routine(void *arg)
     // ------------------------------------------------
     // Initialize HW for FSP Comm
     // ------------------------------------------------
-    l_errlHndl = cmdh_fsp_init();
-    if(l_errlHndl)
-    {
-        // Mark Errl as committed, so FSP knows right away we are having
-        // problems with Attention, if that is the cause of the error.
+    cmdh_comm_init();
+    CHECKPOINT(COMM_INIT_COMPLETED);
 
-        commitErrl(&l_errlHndl);
-    }
-    else    // Mark the Checkpoint only if no error was logged
-    {
-        CHECKPOINT(FSP_COMM_INITIALIZED);
-    }
+    // Read/trace push queue register
+    const uint32_t l_data = in32(OCB_OCBSHCS1);
+    CMDH_TRAC_INFO("Cmd_Hndl_thread_routine: OCB_OCBSHCS1=0x%08X", l_data);
+    // write data after checkpoint and update checkpoint length
+    G_fsp_msg.rsp->fields.data[3] = (l_data >> 24);
+    G_fsp_msg.rsp->fields.data[4] = (l_data >> 16) & 0xFF;
+    G_fsp_msg.rsp->fields.data[5] = (l_data >>  8) & 0xFF;
+    G_fsp_msg.rsp->fields.data[6] = (l_data      ) & 0xFF;
+    G_fsp_msg.rsp->fields.data[7] = 0x11;
+    G_fsp_msg.rsp->fields.data_length[1] = 8; // 8 bytes vs 3
+    dcache_flush_line((void *)CMDH_OCC_RESPONSE_BASE_ADDRESS);
 
     // ------------------------------------------------
     // Loop forever, handling FSP commands
diff --git a/src/occ_405/dcom/dcom.c b/src/occ_405/dcom/dcom.c
index 1016c47..87ae396 100755
--- a/src/occ_405/dcom/dcom.c
+++ b/src/occ_405/dcom/dcom.c
@@ -179,6 +179,8 @@ void dcom_initialize_roles(void)
                          ERRL_CALLOUT_PRIORITY_LOW);
 
         G_pbax_id.valid   = 0;  // Invalid Chip/Node ID
+
+        CHECKPOINT_FAIL_AND_HALT(l_errl);
     }
 
 // Initialize DCOM Thread Sem
@@ -237,7 +239,7 @@ void dcom_initialize_pbax_queues(void)
                 break;
             }
 
-            // create pbax rx queue o
+            // create pbax rx queue 0
             l_rc = pbax_queue_create( &G_pbax_read_queue[0],//queue
                     ASYNC_ENGINE_PBAX_PUSH0,                //engine
                     G_pbax_queue_rx0_buffer,                //cq base
@@ -301,8 +303,7 @@ void dcom_initialize_pbax_queues(void)
             0                                   //Userdata2
             );
 
-        // Commit log and request reset
-        REQUEST_RESET(l_errl);
+        CHECKPOINT_FAIL_AND_HALT(l_errl);
     }
 }
 
diff --git a/src/occ_405/incl/occ_common.h b/src/occ_405/incl/occ_common.h
index ceb3757..2ab096c 100755
--- a/src/occ_405/incl/occ_common.h
+++ b/src/occ_405/incl/occ_common.h
@@ -232,6 +232,14 @@ do {                                                                           \
     .endm
 #endif /* __ASSEMBLER__ */
 
+// Commit specified error and then halt the 405 with Initialization Failure
+#define CHECKPOINT_FAIL_AND_HALT(elog)  \
+{                                                           \
+    setErrlActions(elog, ERRL_ACTIONS_RESET_REQUIRED);      \
+    commitErrl(&elog);                                      \
+    OCC_HALT(ERRL_RC_OCC_INIT_FAILURE);                     \
+}
+
 // Unique checkpoints
 enum
 {
@@ -261,8 +269,9 @@ enum
     CMDH_THREAD_STARTED         = 0x0dff,
     INIT_OCB                    = 0x0e05,
     OCB_INITIALIZED             = 0x0e07,
-    FSP_COMM_INITIALIZED        = 0x0eff,
+    COMM_INIT_COMPLETED         = 0x0eff,
     ABOUT_TO_HALT               = 0x0f00,
+    COMM_INIT_FAILURE           = 0xeeff,
 };
 
 // Checkpoint flags (one byte bitmap)
diff --git a/src/occ_405/main.c b/src/occ_405/main.c
index ff2f3d9..89cabb9 100755
--- a/src/occ_405/main.c
+++ b/src/occ_405/main.c
@@ -735,7 +735,7 @@ void initMainThrdSemAndTimer()
                                       l_hmonSemRc,                  //userdata1
                                       l_timerRc);                   //userdata2
 
-        REQUEST_RESET(l_err);
+        CHECKPOINT_FAIL_AND_HALT(l_err);
     }
 }
 
@@ -764,8 +764,6 @@ void Main_thread_routine(void *private)
     // Default role initialization and determine OCC/Chip Id
 
     dcom_initialize_roles();
-
-
     CHECKPOINT(ROLES_INITIALIZED);
 
     // Sensor Initialization
diff --git a/src/occ_405/proc/proc_data.c b/src/occ_405/proc/proc_data.c
index 09073f9..0952bbf 100755
--- a/src/occ_405/proc/proc_data.c
+++ b/src/occ_405/proc/proc_data.c
@@ -351,8 +351,7 @@ void proc_core_init( void )
                     0                                        //Userdata2
             );
 
-            // commit error log
-            REQUEST_RESET(l_err);
+            CHECKPOINT_FAIL_AND_HALT(l_err);
             break;
         }
 
@@ -390,8 +389,7 @@ void proc_core_init( void )
                                 0                                         //Userdata2
             );
 
-            // commit error log
-            REQUEST_RESET(l_err);
+            CHECKPOINT_FAIL_AND_HALT(l_err);
             break;
         }
 
diff --git a/src/occ_405/thread/threadSch.c b/src/occ_405/thread/threadSch.c
index 7347eab..9c7c660 100755
--- a/src/occ_405/thread/threadSch.c
+++ b/src/occ_405/thread/threadSch.c
@@ -204,16 +204,18 @@ void initThreadScheduler(void)
          * @userdata4   OCC_NO_EXTENDED_RC
          * @devdesc     SSX thread related failure
          */
-        errlHndl_t l_rc = createErrl(THRD_MID_INIT_THREAD_SCHDLR,   // ModId
-                                     SSX_GENERIC_FAILURE,           // Reasoncode
-                                     OCC_NO_EXTENDED_RC,            // Extended reasoncode
-                                     ERRL_SEV_UNRECOVERABLE,        // Severity
-                                     l_trace,                       // Trace Buf
-                                     DEFAULT_TRACE_SIZE,            // Trace Size
-                                     l_timerRc,                     // Userdata1
-                                     l_snapshotTimerRc);            // Userdata2
-
-        REQUEST_RESET(l_rc);
+        errlHndl_t l_errl = createErrl(THRD_MID_INIT_THREAD_SCHDLR,   // ModId
+                                       SSX_GENERIC_FAILURE,           // Reasoncode
+                                       OCC_NO_EXTENDED_RC,            // Extended reasoncode
+                                       ERRL_SEV_UNRECOVERABLE,        // Severity
+                                       l_trace,                       // Trace Buf
+                                       DEFAULT_TRACE_SIZE,            // Trace Size
+                                       l_timerRc,                     // Userdata1
+                                       l_snapshotTimerRc);            // Userdata2
+
+        CHECKPOINT(COMM_INIT_FAILURE);
+
+        REQUEST_RESET(l_errl);
     }
 }