OCC Active state changes required for Simics

Change-Id: Ic6e65bcddaefd5213042707feffcf1272133e532
RTC: 161266
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/31190
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Martha Broyles <mbroyles@us.ibm.com>
Reviewed-by: William A. Bryan <wilbryan@us.ibm.com>

14 files changed, 335 insertions, 203 deletions

diff --git a/src/occ_405/amec/amec_health.c b/src/occ_405/amec/amec_health.c
index 81f88fc..8ad5e0a 100755
--- a/src/occ_405/amec/amec_health.c
+++ b/src/occ_405/amec/amec_health.c
@@ -36,6 +36,7 @@
 //*************************************************************************/
 // Externs
 //*************************************************************************/
+extern bool G_simics_environment;
 
 //*************************************************************************/
 // Defines/Enums
@@ -351,10 +352,12 @@ void amec_health_check_dimm_timeout()
                     l_fru->sample_age = -1;
                 }
 
-                //info trace each transition to not having a new temperature
-                if(l_fru->sample_age == 1)
+                // In Simics: the RTL timer is increased and a DIMM reading will not always
+                // complete on each call.  (an error will still be logged if reading does not
+                // meet the DIMM MAX_READ_TIMEOUT.)
+                if((l_fru->sample_age == 1) && (!G_simics_environment))
                 {
-                    TRAC_INFO("Failed to read dimm temperature on cent[%d] dimm[%d] temp[%d] flags[0x%02X]",
+                    TRAC_INFO("No new DIMM temperature available on cent[%d] dimm[%d] temp[%d] flags[0x%02X]",
                               l_cent, l_dimm, l_fru->cur_temp, l_fru->flags);
                 }
 
@@ -468,8 +471,9 @@ void amec_health_check_dimm_timeout()
                 //clear timer
                 l_fru->sample_age = 0;
 
-                //info trace each time we recover
-                if(L_ran_once)
+                // In Simics: the RTL timer is increased and a DIMM reading will not always
+                // complete on each call.  Skip the "recovery" trace in Simics.
+                if((L_ran_once) && (!G_simics_environment))
                 {
                     TRAC_INFO("DIMM temperature collection has resumed on cent/port[%d] dimm[%d] temp[%d]",
                               l_cent, l_dimm, l_fru->cur_temp);
diff --git a/src/occ_405/cmdh/cmdh_fsp_cmds.c b/src/occ_405/cmdh/cmdh_fsp_cmds.c
index 93bd66b..91a7a37 100755
--- a/src/occ_405/cmdh/cmdh_fsp_cmds.c
+++ b/src/occ_405/cmdh/cmdh_fsp_cmds.c
@@ -1083,7 +1083,11 @@ void cmdh_dbug_cmd (const cmdh_fsp_cmd_t * i_cmd_ptr,
     return;
 }
 
-
+// TODO: Workaround - SSX_SECONDS uses SsxInterval (uint32) instead of SsxTimebase (uint64)
+// That would only allow times up to 7.1 seconds.  OCC has a 15 second timeout for the
+// SET_MODE_AND_STATE command, so updating macro to handle this longer timeout. (waiting for SSX change)
+#undef SSX_SECONDS
+#define SSX_SECONDS(s) ((SsxTimebase)(__ssx_timebase_frequency_hz * (SsxTimebase)(s)))
 // Function Specification
 //
 // Name:  SMGR_base_setmodestate_cmdh
@@ -1097,7 +1101,7 @@ errlHndl_t cmdh_tmgt_setmodestate(const cmdh_fsp_cmd_t * i_cmd_ptr,
     errlHndl_t                      l_errlHndl     = NULL;
     smgr_setmodestate_v0_query_t*   l_cmd_ptr      = (smgr_setmodestate_v0_query_t *)i_cmd_ptr;
     ERRL_RC                         l_rc           = ERRL_RC_INTERNAL_FAIL;
-    SsxInterval                     l_timeout      = SSX_SECONDS(15);
+    SsxTimebase                     l_timeout      = SSX_SECONDS(15);
     SsxTimebase                     l_start        = ssx_timebase_get();
     OCC_STATE                       l_pre_state    = CURRENT_STATE();
     OCC_MODE                        l_pre_mode     = CURRENT_MODE();
@@ -1213,7 +1217,7 @@ errlHndl_t cmdh_tmgt_setmodestate(const cmdh_fsp_cmd_t * i_cmd_ptr,
                 if ( ((ssx_timebase_get() - l_start) > l_timeout))
                 {
                     CMDH_TRAC_ERR("cmdh_tmgt_setmodestate: time out waiting for all slave occ (expected:%d, passed:%d)",
-                             l_occ_num, l_occ_passed_num);
+                                  l_occ_num, l_occ_passed_num);
                     /* @
                      * @errortype
                      * @moduleid    CMDH_GENERIC_CMD_FAILURE
diff --git a/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c b/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
index 913d45e..bc9dabe 100755
--- a/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
+++ b/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
@@ -667,9 +667,10 @@ void apss_store_ipmi_sensor_id(const uint16_t i_channel, const apss_cfg_adc_v20_
     //the channel is not being utilized.
     if ((i_channel < MAX_APSS_ADC_CHANNELS) && (i_adc->assignment != ADC_RESERVED))
     {
-        if (i_adc->ipmisensorId == 0)
+        if ((i_adc->ipmisensorId == 0) && (G_occ_interrupt_type != FSP_SUPPORTED_OCC))
         {
-            CMDH_TRAC_ERR("apss_store_ipmi_sensor_id: Missing Sensor ID for channel %i.",i_channel);
+            // Sensor IDs are not required and only used for BMC based systems
+            CMDH_TRAC_INFO("apss_store_ipmi_sensor_id: No Sensor ID for channel %i.",i_channel);
             //We need to generate a generic sensor ID if we want channels with functionIDs but
             //no sensor IDs to be reported in the poll command.
         }
diff --git a/src/occ_405/dcom/dcomMasterTx.c b/src/occ_405/dcom/dcomMasterTx.c
index 58af809..08edef8 100644
--- a/src/occ_405/dcom/dcomMasterTx.c
+++ b/src/occ_405/dcom/dcomMasterTx.c
@@ -58,6 +58,11 @@ uint32_t G_pbax_packet = 0xffffffff;
 // Used to keep count of number of APSS data collection fails.
 uint16_t G_apss_fail_updown_count = 0x0000;
 
+#ifdef DEBUG_APSS_SEQ
+// used to keep track of the APSS complete sequence number
+extern uint32_t G_savedCompleteSeq;
+#endif
+
 // Function Specification
 //
 // Name: dcom_build_slv_inbox
@@ -258,10 +263,15 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
         // Did APSS power complete?
         if( l_pwr_meas == TRUE )
         {
-#ifdef DCOM_DEBUG
+#if !defined(DEBUG_APSS_SEQ) && defined(DCOM_DEBUG)
             uint64_t l_end = ssx_timebase_get();
             DCOM_DBG("Got APSS after waiting %d us\n",(int)( (l_end-l_start) / ( SSX_TIMEBASE_FREQUENCY_HZ / 1000000 ) ));
 #endif
+#if defined(DEBUG_APSS_SEQ) && !defined(DCOM_DEBUG)
+            uint64_t l_end = ssx_timebase_get();
+            DCOM_DBG("Got APSS after waiting %d us (complete seq %d)\n",
+                     (int)( (l_end-l_start) / ( SSX_TIMEBASE_FREQUENCY_HZ / 1000000 ) ), G_savedCompleteSeq);
+#endif
 
             APSS_SUCCESS();
 
@@ -371,6 +381,7 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
 
                 // Request created at least once
                 L_bce_slv_inbox_tx_request_created_once = TRUE;
+                DCOM_DBG("4.1.1 Scheduling G_slv_inbox_tx_pba_request");
                 l_ssxrc = bce_request_schedule(&G_slv_inbox_tx_pba_request); // Actual copying
 
                 if(l_ssxrc != SSX_OK)
@@ -397,9 +408,7 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
         else
         {
             // check time and break out if we reached limit
-            // TODO: shrink this later depending on how much
-            // work we are doing in RTL
-            if ((ssx_timebase_get() - l_start) < SSX_MICROSECONDS(150))
+            if ((ssx_timebase_get() - l_start) < SSX_MICROSECONDS(DCOM_TX_APSS_WAIT_TIME))
             {
                 continue;
             }
diff --git a/src/occ_405/dcom/dcomSlaveRx.c b/src/occ_405/dcom/dcomSlaveRx.c
index f608291..3c93a89 100644
--- a/src/occ_405/dcom/dcomSlaveRx.c
+++ b/src/occ_405/dcom/dcomSlaveRx.c
@@ -122,7 +122,7 @@ void task_dcom_rx_slv_inbox( task_t *i_self)
 
 #ifdef DCOM_DEBUG
             uint64_t l_end = ssx_timebase_get();
-            DCOM_DBG("Got Doorbell from Master after waiting %d us\n",(int)( (l_end-l_start) / ( SSX_TIMEBASE_FREQUENCY_HZ / 1000000 ) ));
+            DCOM_DBG("1.1 Got Doorbell from Master after waiting %d us\n",(int)( (l_end-l_start) / ( SSX_TIMEBASE_FREQUENCY_HZ / 1000000 ) ));
 #endif
             G_dcomTime.slave.doorbellStopWaitRx = ssx_timebase_get();
             uint64_t l_delta = G_dcomTime.slave.doorbellStopWaitRx - G_dcomTime.slave.doorbellStartWaitRx;
@@ -288,8 +288,9 @@ void task_dcom_rx_slv_inbox( task_t *i_self)
             {
                TRAC_INFO("Only got %d bytes from master",l_bytes);
             }
+
             // Check time and break out if we reached limit
-            if ((ssx_timebase_get() - l_start) < SSX_MICROSECONDS(100))    // TODO: Shrink this down
+            if ((ssx_timebase_get() - l_start) < SSX_MICROSECONDS(100))
             {
                 continue;
             }
@@ -352,7 +353,7 @@ uint32_t dcom_rx_slv_inbox_doorbell( void )
                 &l_read
                 );
 
-        DCOM_DBG("Doorbell (Multicast) Read: %d bytes\n",l_read);
+        DCOM_DBG("1.0.1 Doorbell (Multicast) Read: %d bytes\n",l_read);
 
         // We got an error reading from the PBAX, return to caller
         if ( l_pbarc != 0 )
@@ -374,9 +375,11 @@ uint32_t dcom_rx_slv_inbox_doorbell( void )
             {
                 if(l_bytes_so_far){
                     G_dcomTime.slave.doorbellErrorFlags.incomplete = 1;
+                    DCOM_DBG("dcom_rx_slv_inbox_doorbell: incomplete data");
                 }
                 else{
                     G_dcomTime.slave.doorbellErrorFlags.timeout = 1;
+                    DCOM_DBG("dcom_rx_slv_inbox_doorbell: timeout");
                 }
                 break;
             }
@@ -484,7 +487,7 @@ void task_dcom_wait_for_master( task_t *i_self)
         {
             if (L_first_doorbell_rcvd)
             {
-                // We didn't get a doorbell from the Master, increment our 
+                // We didn't get a doorbell from the Master, increment our
                 // counter
                 L_no_master_doorbell_cnt++;
 
@@ -547,7 +550,7 @@ void task_dcom_wait_for_master( task_t *i_self)
                     }
                 }
 
-                if (L_no_master_doorbell_cnt == APSS_DATA_FAIL_MAX) 
+                if (L_no_master_doorbell_cnt == APSS_DATA_FAIL_MAX)
                 {
                     // If we still don't get a doorbell from the Master for this
                     // long, we will request a reset
diff --git a/src/occ_405/dcom/dcom_thread.c b/src/occ_405/dcom/dcom_thread.c
index f6eda92..93da66b 100755
--- a/src/occ_405/dcom/dcom_thread.c
+++ b/src/occ_405/dcom/dcom_thread.c
@@ -53,9 +53,6 @@ SsxSemaphore G_dcomThreadWakeupSem;
 //              happen more often than the 1-second that other threads run
 //              at.
 //
-//              This thread currently runs ~1ms, based on the RTL loop of
-//              250us.
-//
 //              FWIW -- It is pointless to set this thread to run any more
 //              often than the length of the RTL loop, since it is acting
 //              on data passed back and forth via that loop.
@@ -68,16 +65,15 @@ void Dcom_thread_routine(void *arg)
     SsxTimer  l_timeout_timer;
     errlHndl_t l_errlHndl = NULL;
     // --------------------------------------------------
-    // Create a timer that pops every 10 seconds to wake up
+    // Create a timer that pops every 7 seconds to wake up
     // this thread, in case a semaphore never gets posted.
-    // TODO: Is this really needed?
     // --------------------------------------------------
     ssx_timer_create(&l_timeout_timer,
                      (SsxTimerCallback) ssx_semaphore_post,
                      (void *) &G_dcomThreadWakeupSem);
     ssx_timer_schedule(&l_timeout_timer,
-                       SSX_SECONDS(10),
-                       SSX_SECONDS(10));
+                       SSX_SECONDS(7),
+                       SSX_SECONDS(7));
 
     DCOM_TRAC_INFO("DCOM Thread Started");
     for(;;)
@@ -168,9 +164,9 @@ void Dcom_thread_routine(void *arg)
         // SSX Sleep
         // --------------------------------------------------
         // Even if semaphores are continually posted, there is no reason
-        // for us to run this thread any more often than once every 250us
+        // for us to run this thread any more often than once every tick
         // so we don't starve any other thread
-        ssx_sleep(SSX_MICROSECONDS(250));
+        ssx_sleep(SSX_MICROSECONDS(MICS_PER_TICK));
     }
 }
 
diff --git a/src/occ_405/dimm/dimm.c b/src/occ_405/dimm/dimm.c
index 676f127..567fca5 100755
--- a/src/occ_405/dimm/dimm.c
+++ b/src/occ_405/dimm/dimm.c
@@ -138,20 +138,21 @@ void memory_nimbus_init()
     int rc_dimm_sm = 0, rc_dimm_ctl = 0; // rcs from gpe ipc request creation commands
     int rc_dimm = 0;
 
-    do {
+    do
+    {
         // Initializes GPE request for DIMM temperature reading IPC Task,
         DIMM_DBG("memory_nimbus_init: Creating request GPE1 DIMM data IPC task");
         rc_dimm_sm = gpe_request_create(
-            &G_dimm_sm_request,              // gpe_req for the task
-            &G_async_gpe_queue1,             // queue
-            IPC_ST_DIMM_SM_FUNCID,           // Function ID
-            &G_dimm_sm_args,                 // parm for the task
-            SSX_WAIT_FOREVER,                // no timeout
-            NULL,                            // callback
-            NULL,                            // callback arg
-            ASYNC_CALLBACK_IMMEDIATE);       // options
+                                        &G_dimm_sm_request,              // gpe_req for the task
+                                        &G_async_gpe_queue1,             // queue
+                                        IPC_ST_DIMM_SM_FUNCID,           // Function ID
+                                        &G_dimm_sm_args,                 // parm for the task
+                                        SSX_WAIT_FOREVER,                // no timeout
+                                        NULL,                            // callback
+                                        NULL,                            // callback arg
+                                        ASYNC_CALLBACK_IMMEDIATE);       // options
         if(rc_dimm_sm)
-    {
+        {
             TRAC_ERR("memory_control_init: Failed to initialize dimm state"
                      " machine IPC task [rc_dimm_sm=0x%x]",
                      rc_dimm_sm);
@@ -162,14 +163,14 @@ void memory_nimbus_init()
         // Initializes GPE request for DIMM Control IPC Task,
         DIMM_DBG("memory_nimbus_init: Creating request GPE1 DIMM control IPC task");
         rc_dimm_ctl = gpe_request_create(
-            &G_memory_control_task.gpe_req,  // gpe_req for the task
-            &G_async_gpe_queue1,             // queue
-            IPC_ST_DIMM_CONTROL_FUNCID,      // Function ID
-            &G_dimm_control_args,            // parm for the task
-            SSX_WAIT_FOREVER,                // no timeout
-            NULL,                            // callback
-            NULL,                            // callback argument
-            ASYNC_CALLBACK_IMMEDIATE );      // options
+                                         &G_memory_control_task.gpe_req,  // gpe_req for the task
+                                         &G_async_gpe_queue1,             // queue
+                                         IPC_ST_DIMM_CONTROL_FUNCID,      // Function ID
+                                         &G_dimm_control_args,            // parm for the task
+                                         SSX_WAIT_FOREVER,                // no timeout
+                                         NULL,                            // callback
+                                         NULL,                            // callback argument
+                                         ASYNC_CALLBACK_IMMEDIATE );      // options
 
         if(rc_dimm_ctl)
         {
@@ -179,31 +180,31 @@ void memory_nimbus_init()
             rc_dimm = rc_dimm_sm;
             break;
         }
-            }
+    }
     while(0);
 
 
     if(rc_dimm) // Either dimm sm or dimm control gpe request creation failed.
-            {
+    {
         /* @
-                 * @errortype
+         * @errortype
          * @moduleid    DIMM_MID_NIMBUS_INIT
          * @reasoncode  SSX_GENERIC_FAILURE
          * @userdata1   l_rc_gpe  - Return code of failing function
-                 * @userdata2   0
+         * @userdata2   0
          * @userdata4   ERC_CENTAUR_GPE_REQUEST_CREATE_FAILURE
          * @devdesc     Failed to initialize GPE1 DIMM IPC job
-                 */
+         */
         l_err = createErrl(
-            DIMM_MID_NIMBUS_INIT,                       //modId
-            SSX_GENERIC_FAILURE,                        //reasoncode
-            OCC_NO_EXTENDED_RC,                         //Extended reason code
-            ERRL_SEV_PREDICTIVE,                        //Severity
-            NULL,                                       //Trace Buf
-            DEFAULT_TRACE_SIZE,                         //Trace Size
-            rc_dimm,                                    //userdata1
-            0                                           //userdata2
-            );
+                           DIMM_MID_NIMBUS_INIT,                       //modId
+                           SSX_GENERIC_FAILURE,                        //reasoncode
+                           OCC_NO_EXTENDED_RC,                         //Extended reason code
+                           ERRL_SEV_PREDICTIVE,                        //Severity
+                           NULL,                                       //Trace Buf
+                           DEFAULT_TRACE_SIZE,                         //Trace Size
+                           rc_dimm,                                    //userdata1
+                           0                                           //userdata2
+                          );
 
         REQUEST_RESET(l_err);
     }
@@ -333,8 +334,8 @@ bool schedule_dimm_req(uint8_t i_state, dimm_sm_args_t i_new_args)
 
     if (!async_request_is_idle(&G_dimm_sm_request.request))
     {
-        INTR_TRAC_ERR("schedule_dimm_req: request not idle when scheduling 0x%02X (tick=%d) (prior state 0x%02X / DIMM%04X)",
-                      i_state, DIMM_TICK, G_dimm_sm_args.state, DIMM_AND_PORT);
+        INTR_TRAC_INFO("E>schedule_dimm_req: request not idle when scheduling 0x%02X (tick=%d) (prior state 0x%02X / DIMM%04X)",
+                       i_state, DIMM_TICK, G_dimm_sm_args.state, DIMM_AND_PORT);
     }
     else
     {
diff --git a/src/occ_405/main.c b/src/occ_405/main.c
index 75c4476..dca5f89 100755
--- a/src/occ_405/main.c
+++ b/src/occ_405/main.c
@@ -134,6 +134,12 @@ void check_runtime_environment(void)
     //       a backdoor hack in Simics to set bit 63, telling the
     //       firmware what environment it's running in.
     G_simics_environment = ( 0 != (flags & 0x0000000000000001) ) ? TRUE : FALSE;
+    if (G_simics_environment)
+    {
+        // slow down RTL for Simics
+        G_mics_per_tick = SIMICS_MICS_PER_TICK;
+        G_dcom_tx_apss_wait_time = SIMICS_MICS_PER_TICK * 6 / 10;
+    }
 }
 
 /*
@@ -1142,8 +1148,14 @@ int main(int argc, char **argv)
 
     MAIN_TRAC_INFO("Inside OCC Main");
 
-    MAIN_TRAC_INFO("Currently %srunning in Simics environment",
-                  ((G_simics_environment == FALSE) ? "not " : "") );
+    if (G_simics_environment == FALSE)
+    {
+        MAIN_TRAC_INFO("Currently not running in Simics environment");
+    }
+    else
+    {
+        MAIN_TRAC_INFO("Currently running in Simics environment");
+    }
 
     // Trace what happened before ssx initialization
     MAIN_TRAC_INFO("HOMER accessed, rc=%d, version=%d, ssx_rc=%d",
diff --git a/src/occ_405/pss/apss.c b/src/occ_405/pss/apss.c
index 5529a11..e93b156 100755
--- a/src/occ_405/pss/apss.c
+++ b/src/occ_405/pss/apss.c
@@ -70,6 +70,13 @@ GpeRequest G_meas_complete_request;
 // Up / down counter for redundant apss failures
 uint32_t G_backup_fail_count = 0;
 
+#ifdef DEBUG_APSS_SEQ
+uint32_t G_sequence_start = 0;
+uint32_t G_sequence_cont = 0;
+uint32_t G_sequence_complete = 0;
+volatile uint32_t G_savedCompleteSeq = 0;
+#endif
+
 // Used to tell slave inbox that pwr meas is complete
 volatile bool G_ApssPwrMeasCompleted = FALSE;
 
@@ -320,11 +327,15 @@ void task_apss_start_pwr_meas(struct task *i_self)
         {
             if (!L_idle_traced)
             {
-                INTR_TRAC_ERR("task_apss_start_pwr_meas: request is not idle.");
+                INTR_TRAC_INFO("E>task_apss_start_pwr_meas: request is not idle.");
                 L_idle_traced = TRUE;
             }
             break;
         }
+        else
+        {
+            L_idle_traced = FALSE;
+        }
 
         // Check if we need to try recovering the apss
         if(G_apss_recovery_requested)
@@ -392,40 +403,62 @@ void task_apss_start_pwr_meas(struct task *i_self)
         G_gpe_start_pwr_meas_read_args.error.error = 0;
         G_gpe_start_pwr_meas_read_args.error.ffdc = 0;
 
-        // Submit the next request
-        l_rc = gpe_request_schedule(&G_meas_start_request);
-        if (0 != l_rc)
+#ifdef DEBUG_APSS_SEQ
+        // DEBUG: Only allow start if the previous complete was scheduled
+        static bool L_throttle = FALSE;
+        if (G_sequence_complete == G_sequence_start)
         {
-            errlHndl_t l_err = NULL;
+            L_throttle = FALSE;
+            ++G_sequence_start;
+            TRAC_INFO("task_apss_start_pwr_meas: scheduling start (seq %d)", G_sequence_start);
+#endif
 
-            INTR_TRAC_ERR("task_apss_start_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
-                     (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
+            // Submit the next request
+            l_rc = gpe_request_schedule(&G_meas_start_request);
+            if (0 != l_rc)
+            {
+                errlHndl_t l_err = NULL;
 
-            /*
-             * @errortype
-             * @moduleid    PSS_MID_APSS_START_MEAS
-             * @reasoncode  SSX_GENERIC_FAILURE
-             * @userdata1   GPE shedule returned rc code
-             * @userdata2   0
-             * @userdata4   ERC_APSS_SCHEDULE_FAILURE
-             * @devdesc     task_apss_start_pwr_meas schedule failed
-             */
-            l_err = createErrl(PSS_MID_APSS_START_MEAS,
-                               SSX_GENERIC_FAILURE,
-                               ERC_APSS_SCHEDULE_FAILURE,
-                               ERRL_SEV_PREDICTIVE,
-                               NULL,
-                               DEFAULT_TRACE_SIZE,
-                               l_rc,
-                               0);
-
-            // Request reset since this should never happen.
-            REQUEST_RESET(l_err);
-            L_scheduled = FALSE;
-            break;
-        }
+                INTR_TRAC_ERR("task_apss_start_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
+                              (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
 
-        L_scheduled = TRUE;
+                /*
+                 * @errortype
+                 * @moduleid    PSS_MID_APSS_START_MEAS
+                 * @reasoncode  SSX_GENERIC_FAILURE
+                 * @userdata1   GPE shedule returned rc code
+                 * @userdata2   0
+                 * @userdata4   ERC_APSS_SCHEDULE_FAILURE
+                 * @devdesc     task_apss_start_pwr_meas schedule failed
+                 */
+                l_err = createErrl(PSS_MID_APSS_START_MEAS,
+                                   SSX_GENERIC_FAILURE,
+                                   ERC_APSS_SCHEDULE_FAILURE,
+                                   ERRL_SEV_PREDICTIVE,
+                                   NULL,
+                                   DEFAULT_TRACE_SIZE,
+                                   l_rc,
+                                   0);
+
+                // Request reset since this should never happen.
+                REQUEST_RESET(l_err);
+                L_scheduled = FALSE;
+                break;
+            }
+
+            L_scheduled = TRUE;
+#ifdef DEBUG_APSS_SEQ
+        }
+        else
+        {
+            if (!L_throttle)
+            {
+                TRAC_INFO("task_apss_start_pwr_meas: start seq(%d) != complete seq(%d) so skipping start",
+                          G_sequence_start, G_sequence_complete);
+                L_throttle = TRUE;
+            }
+        }
+#endif
 
 
     }while (0);
@@ -470,11 +503,15 @@ void task_apss_continue_pwr_meas(struct task *i_self)
         {
             if (!L_idle_traced)
             {
-                INTR_TRAC_ERR("task_apss_continue_pwr_meas: request is not idle.");
+                INTR_TRAC_INFO("E>task_apss_continue_pwr_meas: request is not idle.");
                 L_idle_traced = TRUE;
             }
             break;
         }
+        else
+        {
+            L_idle_traced = FALSE;
+        }
 
         //Don't run anything if apss recovery is in progress
         if(G_apss_recovery_requested)
@@ -539,40 +576,63 @@ void task_apss_continue_pwr_meas(struct task *i_self)
         G_gpe_continue_pwr_meas_read_args.error.error = 0;
         G_gpe_continue_pwr_meas_read_args.error.ffdc = 0;
 
-        // Submit the next request
-        l_rc = gpe_request_schedule(&G_meas_cont_request);
-        if (0 != l_rc)
+#ifdef DEBUG_APSS_SEQ
+        // DEBUG: Only allow new continue if the start was scheduled
+        static bool L_throttle = FALSE;
+        if (G_sequence_start == (G_sequence_cont+1))
         {
-            errlHndl_t l_err = NULL;
+            L_throttle = FALSE;
+            ++G_sequence_cont;
+            TRAC_INFO("task_apss_cont_pwr_meas: scheduling cont (seq %d)", G_sequence_cont);
+#endif
 
-            INTR_TRAC_ERR("task_apss_cont_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
-                     (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
+            // Submit the next request
+            l_rc = gpe_request_schedule(&G_meas_cont_request);
+            if (0 != l_rc)
+            {
+                errlHndl_t l_err = NULL;
 
-            /*
-             * @errortype
-             * @moduleid    PSS_MID_APSS_CONT_MEAS
-             * @reasoncode  SSX_GENERIC_FAILURE
-             * @userdata1   GPE shedule returned rc code
-             * @userdata2   0
-             * @userdata4   ERC_APSS_SCHEDULE_FAILURE
-             * @devdesc     task_apss_continue_pwr_meas schedule failed
-             */
-            l_err = createErrl(PSS_MID_APSS_CONT_MEAS,
-                               SSX_GENERIC_FAILURE,
-                               ERC_APSS_SCHEDULE_FAILURE,
-                               ERRL_SEV_PREDICTIVE,
-                               NULL,
-                               DEFAULT_TRACE_SIZE,
-                               l_rc,
-                               0);
-
-            // Request reset since this should never happen.
-            REQUEST_RESET(l_err);
-            L_scheduled = FALSE;
-            break;
-        }
+                INTR_TRAC_ERR("task_apss_cont_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
+                              (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
 
-        L_scheduled = TRUE;
+                /*
+                 * @errortype
+                 * @moduleid    PSS_MID_APSS_CONT_MEAS
+                 * @reasoncode  SSX_GENERIC_FAILURE
+                 * @userdata1   GPE shedule returned rc code
+                 * @userdata2   0
+                 * @userdata4   ERC_APSS_SCHEDULE_FAILURE
+                 * @devdesc     task_apss_continue_pwr_meas schedule failed
+                 */
+                l_err = createErrl(PSS_MID_APSS_CONT_MEAS,
+                                   SSX_GENERIC_FAILURE,
+                                   ERC_APSS_SCHEDULE_FAILURE,
+                                   ERRL_SEV_PREDICTIVE,
+                                   NULL,
+                                   DEFAULT_TRACE_SIZE,
+                                   l_rc,
+                                   0);
+
+                // Request reset since this should never happen.
+                REQUEST_RESET(l_err);
+                L_scheduled = FALSE;
+                break;
+            }
+
+            L_scheduled = TRUE;
+
+#ifdef DEBUG_APSS_SEQ
+        }
+        else
+        {
+            if (!L_throttle)
+            {
+                TRAC_INFO("task_apss_cont_pwr_meas: start seq(%d) != cont seq(%d+1) so skipping cont",
+                          G_sequence_start, G_sequence_cont);
+                L_throttle = TRUE;
+            }
+        }
+#endif
 
     }while (0);
 
@@ -590,13 +650,11 @@ void task_apss_continue_pwr_meas(struct task *i_self)
 //              collection for this loop.
 //
 // End Function Specification
-#define APSS_ADC_SEQ_MASK 0xf000f000f000f000ull
+#define APSS_ADC_SEQ_MASK  0xf000f000f000f000ull
 #define APSS_ADC_SEQ_CHECK 0x0000100020003000ull
 void reformat_meas_data()
 {
-    APSS_DBG("GPE_complete_pwr_meas_read finished w/rc=0x%08X\n", G_gpe_complete_pwr_meas_read_args.error.rc);
-    APSS_DBG_HEXDUMP(&G_gpe_complete_pwr_meas_read_args, sizeof(G_gpe_complete_pwr_meas_read_args), "G_gpe_complete_pwr_meas_read_args");
-
+    // NO TRACING ALLOWED IN CRITICAL INTERRUPT (any IPC callback functions)
     do
     {
         // Make sure complete was successful
@@ -624,15 +682,12 @@ void reformat_meas_data()
         if(G_occ_role == OCC_MASTER)
         {
             // Fail every 16 seconds
-            APSS_DBG("Populate meas data:\n");
-
             // Merge continue/complete data into a single buffer
             const uint16_t l_continue_meas_length = sizeof(G_gpe_continue_pwr_meas_read_args.meas_data);
             const uint16_t l_complete_meas_length = sizeof(G_gpe_complete_pwr_meas_read_args.meas_data);
             uint8_t l_buffer[l_continue_meas_length+l_complete_meas_length];
             memcpy(&l_buffer[                     0], G_gpe_continue_pwr_meas_read_args.meas_data, l_continue_meas_length);
             memcpy(&l_buffer[l_continue_meas_length], G_gpe_complete_pwr_meas_read_args.meas_data, l_complete_meas_length);
-            APSS_DBG_HEXDUMP(l_buffer, sizeof(l_buffer), "l_buffer");
 
             // Copy measurements into correct struction locations (based on composite config)
             uint16_t l_index = 0;
@@ -641,18 +696,17 @@ void reformat_meas_data()
             memcpy(G_apss_pwr_meas.gpio, &l_buffer[l_index], (G_apss_mode_config.numGpioPortsToRead * 2));
             // TOD is always located at same offset
             memcpy(&G_apss_pwr_meas.tod, &l_buffer[l_continue_meas_length+l_complete_meas_length-8], 8);
-
-            APSS_DBG("...into structure: (%d ADC, %d GPIO)\n", G_apss_mode_config.numAdcChannelsToRead,
-                G_apss_mode_config.numGpioPortsToRead);
-            APSS_DBG_HEXDUMP(&G_apss_pwr_meas, sizeof(G_apss_pwr_meas), "G_apss_pwr_meas");
         }
 
         // Mark apss pwr meas completed and valid
         G_ApssPwrMeasCompleted = TRUE;
+#ifdef DEBUG_APSS_SEQ
+        // Save the complete sequence number from the GPE
+        G_savedCompleteSeq = G_gpe_complete_pwr_meas_read_args.error.ffdc;
+#endif
         G_gpe_apss_time_end = ssx_timebase_get();
-        APSS_DBG("APSS Completed - %d\n",(int) ssx_timebase_get());
   }while(0);
-}
+} // end reformat_meas_data()
 
 
 // Note: The complete request must be global, since it must stick around until after the
@@ -684,11 +738,15 @@ void task_apss_complete_pwr_meas(struct task *i_self)
         {
             if (!L_idle_traced)
             {
-                INTR_TRAC_ERR("task_apss_complete_pwr_meas: request is not idle.");
+                INTR_TRAC_INFO("E>task_apss_complete_pwr_meas: request is not idle.");
                 L_idle_traced = TRUE;
             }
             break;
         }
+        else
+        {
+            L_idle_traced = FALSE;
+        }
         if(G_apss_recovery_requested)
         {
             // Allow apss measurement to proceed on next tick
@@ -749,44 +807,72 @@ void task_apss_complete_pwr_meas(struct task *i_self)
             }
         }
 
-        // Clear these out prior to starting the GPE (GPE only sets them)
-        G_gpe_complete_pwr_meas_read_args.error.error = 0;
-        G_gpe_complete_pwr_meas_read_args.error.ffdc = 0;
+#ifdef DEBUG_APSS_SEQ
+        // DEBUG: Only allow new complete if the continue was scheduled
+        static bool L_throttle = FALSE;
+        if (G_sequence_cont == (G_sequence_complete+1))
+        {
+            L_throttle = FALSE;
+            ++G_sequence_complete;
+
+            // Clear these out prior to starting the GPE (GPE only sets them)
+            G_gpe_complete_pwr_meas_read_args.error.error = 0;
+            // Set the FFDC field to the complete sequence number
+            G_gpe_complete_pwr_meas_read_args.error.ffdc = G_sequence_complete;
+            TRAC_INFO("task_apss_complete_pwr_meas: scheduling complete (seq %d, prior seq was %d)",
+                      G_sequence_complete, G_savedCompleteSeq);
+#else
+            // Clear these out prior to starting the GPE (GPE only sets them)
+            G_gpe_complete_pwr_meas_read_args.error.error = 0;
+            G_gpe_complete_pwr_meas_read_args.error.ffdc = 0;
+#endif
 
-        // Submit the next request
+            // Submit the next request
 
-      l_rc = gpe_request_schedule(&G_meas_complete_request);
-        if (0 != l_rc)
-        {
+            l_rc = gpe_request_schedule(&G_meas_complete_request);
+            if (0 != l_rc)
+            {
 
-            errlHndl_t l_err = NULL;
+                errlHndl_t l_err = NULL;
 
-            INTR_TRAC_ERR("task_apss_complete_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
-                     (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
-            /*
-             * @errortype
-             * @moduleid    PSS_MID_APSS_COMPLETE_MEAS
-             * @reasoncode  SSX_GENERIC_FAILURE
-             * @userdata1   GPE shedule returned rc code
-             * @userdata2   0
-             * @userdata4   ERC_APSS_SCHEDULE_FAILURE
-             * @devdesc     task_apss_complete_pwr_meas schedule failed
-             */
-            l_err = createErrl(PSS_MID_APSS_COMPLETE_MEAS,
-                               SSX_GENERIC_FAILURE,
-                               ERC_APSS_SCHEDULE_FAILURE,
-                               ERRL_SEV_PREDICTIVE,
-                               NULL,
-                               DEFAULT_TRACE_SIZE,
-                               l_rc,
-                               0);
-
-            // Request reset since this should never happen.
-            REQUEST_RESET(l_err);
-            L_scheduled = FALSE;
-            break;
+                INTR_TRAC_ERR("task_apss_complete_pwr_meas: schedule failed w/rc=0x%08X (%d us)", l_rc,
+                              (int) ((ssx_timebase_get())/(SSX_TIMEBASE_FREQUENCY_HZ/1000000)));
+                /*
+                 * @errortype
+                 * @moduleid    PSS_MID_APSS_COMPLETE_MEAS
+                 * @reasoncode  SSX_GENERIC_FAILURE
+                 * @userdata1   GPE shedule returned rc code
+                 * @userdata2   0
+                 * @userdata4   ERC_APSS_SCHEDULE_FAILURE
+                 * @devdesc     task_apss_complete_pwr_meas schedule failed
+                 */
+                l_err = createErrl(PSS_MID_APSS_COMPLETE_MEAS,
+                                   SSX_GENERIC_FAILURE,
+                                   ERC_APSS_SCHEDULE_FAILURE,
+                                   ERRL_SEV_PREDICTIVE,
+                                   NULL,
+                                   DEFAULT_TRACE_SIZE,
+                                   l_rc,
+                                   0);
+
+                // Request reset since this should never happen.
+                REQUEST_RESET(l_err);
+                L_scheduled = FALSE;
+                break;
+            }
+            L_scheduled = TRUE;
+#ifdef DEBUG_APSS_SEQ
+        }
+        else
+        {
+            if (!L_throttle)
+            {
+                TRAC_INFO("task_apss_complete_pwr_meas: cont seq(%d) != complete seq(%d+1) so skipping complete",
+                          G_sequence_cont, G_sequence_complete);
+                L_throttle = TRUE;
+            }
         }
-        L_scheduled = TRUE;
+#endif
 
 
     }while (0);
diff --git a/src/occ_405/rtls/rtls.c b/src/occ_405/rtls/rtls.c
index 5788c69..63bb582 100755
--- a/src/occ_405/rtls/rtls.c
+++ b/src/occ_405/rtls/rtls.c
@@ -50,6 +50,12 @@ uint32_t G_run_mask_deferred = GLOBAL_RUN_MASK;
 // The value of the current tick
 uint32_t G_current_tick = 0xFFFFFFFF;
 
+// Number of micro-seconds per tick
+uint32_t G_mics_per_tick = HW_MICS_PER_TICK;
+
+// Number of micro-seconds to wait for APSS data collection to complete
+uint32_t G_dcom_tx_apss_wait_time = HW_MICS_PER_TICK * 0.6;
+
 // The durations measured within the current tick
 fw_timing_t G_fw_timing;
 
diff --git a/src/occ_405/rtls/rtls.h b/src/occ_405/rtls/rtls.h
index a408b19..0aead87 100755
--- a/src/occ_405/rtls/rtls.h
+++ b/src/occ_405/rtls/rtls.h
@@ -107,13 +107,23 @@ typedef struct
                                RTL_FLAG_ACTIVE | RTL_FLAG_MSTR_READY | RTL_FLAG_NO_APSS | \
                                RTL_FLAG_RUN | RTL_FLAG_APSS_NOT_INITD)
 
-// Tick Timer definitions
-#define MICS_PER_TICK 250  // Number of micro-seconds per tick
+// Tick Timer definitions:
+#define MICS_PER_TICK G_mics_per_tick // Number of micro-seconds per tick
 #define MAX_NUM_TICKS 16   // Number of entries in the global tick table (power of 2)
+// MICS_PER_TICK must be lower than 62ms to guarantee the tick table completes in < 1s (for health monitor)
+#define HW_MICS_PER_TICK        250
+#define SIMICS_MICS_PER_TICK  20000 // slow down RTL to 20ms for Simics
+#define DCOM_TX_APSS_WAIT_TIME G_dcom_tx_apss_wait_time
 
 // The value of the current tick
 extern uint32_t G_current_tick;
 
+// The number of micro-seconds per tick
+extern uint32_t G_mics_per_tick;
+
+// The number of micro-seconds to wait for APSS data to complete
+extern uint32_t G_dcom_tx_apss_wait_time;
+
 // The durations measured within the current tick
 extern fw_timing_t G_fw_timing;
 
diff --git a/src/occ_405/state.c b/src/occ_405/state.c
index e0f1310..1314423 100755
--- a/src/occ_405/state.c
+++ b/src/occ_405/state.c
@@ -260,7 +260,6 @@ errlHndl_t SMGR_observation_to_active()
                 l_extRc = ERC_GENERIC_TIMEOUT;
                 break;
             }
-            ssx_sleep(SSX_MICROSECONDS(500));
         }
         if(proc_is_hwpstate_enabled() && G_sysConfigData.system_type.kvm)
         {
diff --git a/src/occ_405/thread/threadSch.c b/src/occ_405/thread/threadSch.c
index 9c7c660..46891c9 100755
--- a/src/occ_405/thread/threadSch.c
+++ b/src/occ_405/thread/threadSch.c
@@ -43,14 +43,12 @@ SsxTimer G_threadSchTimer;
 // Index of highest priority thread in G_scheduledThreads
 uint16_t G_threadSchedulerIndex = 0;
 
-// TEMP: Commented out dcom thread for Simics enablement.
-//       Will need to reenable them when we need them again.
 // Array that holds the threads that need scheduling
 SsxThread* G_scheduledThreads[] =
 {
     &Main_thread,
     &Cmd_Hndl_thread,
-//    &Dcom_thread,
+    &Dcom_thread,
 };
 
 // Error log counter for the callback so that only 1 error log is created
diff --git a/src/occ_gpe0/apss_read.c b/src/occ_gpe0/apss_read.c
index 12e7102..9ffd334 100644
--- a/src/occ_gpe0/apss_read.c
+++ b/src/occ_gpe0/apss_read.c
@@ -55,18 +55,15 @@ void apss_start_pwr_meas_read(ipc_msg_t* cmd, void* arg)
     // the ipc arguments passed through the ipc_msg_t structure, has a pointer
     // to the G_gpe_start_pwr_meas_read_args struct.
 
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_start_pwr_meas_read: enter");
+#endif
+
     int      rc;
     uint64_t regValue; // a pointer to hold the putscom_abs register value
     ipc_async_cmd_t *async_cmd = (ipc_async_cmd_t*)cmd;
     apss_start_args_t *args = (apss_start_args_t*)async_cmd->cmd_data;
 
-    // clear error, ffdc, and rc (feedback to 405)
-    // These may be overwritten by error codes if errors occur
-    // REVIEW: Since the OCC clears these fields, do we really have to repeat this here?
-    args->error.error = 0;
-    args->error.rc = 0;
-    args->error.ffdc = 0;
-
     do{
 
         // wait for ADC completion, or timeout after 5 micro seconds.
@@ -158,11 +155,15 @@ void apss_start_pwr_meas_read(ipc_msg_t* cmd, void* arg)
 
     } while(0);
 
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_start_pwr_meas_read: calling ipc_send_rsp()");
+#endif
+
     // send back a response, IPC success even if ffdc/rc are non zeros
     rc = ipc_send_rsp(cmd, IPC_RC_SUCCESS);
     if(rc)
     {
-        PK_TRACE("apss_start_pwr_meas_read: Failed to send response back. Halting GPE0", rc);
+        PK_TRACE("apss_start_pwr_meas_read: Failed to send response back. Halting GPE0. rc = 0x%08X", rc);
         gpe_set_ffdc(&(args->error), 0x00, GPE_RC_IPC_SEND_FAILED, rc);
         pk_halt();
     }
@@ -195,13 +196,9 @@ void apss_continue_pwr_meas_read(ipc_msg_t* cmd, void* arg)
     ipc_async_cmd_t *async_cmd = (ipc_async_cmd_t*)cmd;
     apss_continue_args_t *args = (apss_continue_args_t*)async_cmd->cmd_data;
 
-    // Clear error, ffdc, and rc (feedback to 405)
-    // These may be overwritten by error codes if errors occur
-    // REVIEW: Since the OCC clears these fields, do we really have to repeat this here?
-    args->error.error = 0;
-    args->error.rc = 0;
-    args->error.ffdc = 0;
-
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_continue_pwr_meas_read: enter");
+#endif
 
     do{
         // wait for ADC completion, or timeout after 100 micro seconds.
@@ -261,11 +258,16 @@ void apss_continue_pwr_meas_read(ipc_msg_t* cmd, void* arg)
         }
     } while(0);
 
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_continue_pwr_meas_read: calling ipc_send_rsp()");
+#endif
+
     // send back a response, IPC success (even if ffdc/rc are non zeros)
     rc = ipc_send_rsp(cmd, IPC_RC_SUCCESS);
     if(rc)
     {
-        PK_TRACE("apss_continue_pwr_meas_read: Failed to send response back. Halting GPE0", rc);
+        PK_TRACE("apss_continue_pwr_meas_read: Failed to send response back. Halting GPE0. rc = 0x%08x",
+                 rc);
         gpe_set_ffdc(&(args->error), 0x00, GPE_RC_IPC_SEND_FAILED, rc);
         pk_halt();
     }
@@ -293,17 +295,14 @@ void apss_complete_pwr_meas_read(ipc_msg_t* cmd, void* arg)
     // the ipc arguments are passed through the ipc_msg_t structure, has a pointer
     // to the G_gpe_complete_pwr_meas_read_args
 
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_complete_pwr_meas_read: enter");
+#endif
+
     int          rc;
     ipc_async_cmd_t *async_cmd = (ipc_async_cmd_t*)cmd;
     apss_complete_args_t *args = (apss_complete_args_t*)async_cmd->cmd_data;
 
-    // clear error, ffdc, and rc (feedback to 405)
-    // These may be overwritten by error codes if errors occur
-    // REVIEW: Since the OCC clears these fields, do we really have to repeat this here?
-    args->error.error = 0;
-    args->error.rc = 0;
-    args->error.ffdc = 0;
-
     do {
         // wait for ADC completion, or timeout after 100 micro seconds.
         // scom register SPIPSS_ADC_STATUS_REG's bit 0 (HWCTRL_ONGOING)
@@ -342,11 +341,15 @@ void apss_complete_pwr_meas_read(ipc_msg_t* cmd, void* arg)
         }
     } while(0);
 
+#ifdef DEBUG_APSS_SEQ
+    PK_TRACE("apss_complete_pwr_meas_read: calling ipc_send_rsp()");
+#endif
+
     // send back a response, IPC success (even if ffdc/rc are non zeros)
     rc = ipc_send_rsp(cmd, IPC_RC_SUCCESS);
     if(rc)
     {
-        PK_TRACE("apss_complete_pwr_meas_read: Failed to send response back. Halting GPE0", rc);
+        PK_TRACE("apss_complete_pwr_meas_read: Failed to send response back. Halting GPE0. rc = 0x%08X", rc);
         gpe_set_ffdc(&(args->error), 0x00, GPE_RC_IPC_SEND_FAILED, rc);
         pk_halt();
     }