Fix excessive traces for BCE requests, slave data collection and opal updates

Correct power cap information in OPAL shared memory

Add non-zero error history counts to poll response

Fix Characterization to Active State change failure

Change-Id: I92b783b631e79786e6190a4def520fee32c2cc7c
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/43216
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christopher J. Cain <cjcain@us.ibm.com>
Reviewed-by: William A. Bryan <wilbryan@us.ibm.com>

16 files changed, 589 insertions, 292 deletions

diff --git a/src/occ_405/cmdh/cmdh_fsp_cmds.c b/src/occ_405/cmdh/cmdh_fsp_cmds.c
index 0933372..84f14bc 100755
--- a/src/occ_405/cmdh/cmdh_fsp_cmds.c
+++ b/src/occ_405/cmdh/cmdh_fsp_cmds.c
@@ -145,6 +145,7 @@ ERRL_RC cmdh_poll_v20(cmdh_fsp_rsp_t * o_rsp_ptr)
 {
     ERRL_RC                     l_rc  = ERRL_RC_INTERNAL_FAIL;
     uint8_t                     k = 0, l_max_sensors = 0;
+    uint8_t                     l_err_hist_idx = 0, l_sens_list_idx = 0;
     cmdh_poll_sensor_db_t       l_sensorHeader;
 
     // Set pointer to start of o_rsp_ptr
@@ -562,7 +563,7 @@ ERRL_RC cmdh_poll_v20(cmdh_fsp_rsp_t * o_rsp_ptr)
     l_sensorHeader.length = sizeof(cmdh_poll_extn_sensor_t);
     l_sensorHeader.count  = 0;
 
-    cmdh_poll_extn_sensor_t l_extnSensorList[4] = {{0}};
+    cmdh_poll_extn_sensor_t l_extnSensorList[MAX_EXTN_SENSORS] = {{0}};
     l_extnSensorList[l_sensorHeader.count].name = EXTN_NAME_FMIN;
     uint16_t freq = G_sysConfigData.sys_mode_freq.table[OCC_MODE_MIN_FREQUENCY];
     l_extnSensorList[l_sensorHeader.count].data[0] = proc_freq2pstate(freq);
@@ -594,6 +595,45 @@ ERRL_RC cmdh_poll_v20(cmdh_fsp_rsp_t * o_rsp_ptr)
     }
     l_sensorHeader.count++;
 
+    // add any non-0 error history counts
+    for(l_err_hist_idx=0; l_err_hist_idx < ERR_HISTORY_SIZE; l_err_hist_idx++)
+    {
+        if(G_error_history[l_err_hist_idx])
+        {
+            if(l_sens_list_idx == 0)
+            {
+                // first one to add fill in name
+                l_extnSensorList[l_sensorHeader.count].name = EXTN_NAME_ERRHIST;
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = l_err_hist_idx;
+                l_sens_list_idx++;
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = G_error_history[l_err_hist_idx];
+                l_sens_list_idx++;
+            }
+            else if(l_sens_list_idx < 5)  // room in current extended error history sensor?
+            {
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = l_err_hist_idx;
+                l_sens_list_idx++;
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = G_error_history[l_err_hist_idx];
+                l_sens_list_idx++;
+            }
+            else // no room start another extended error history sensor
+            {
+                l_sensorHeader.count++;
+                l_extnSensorList[l_sensorHeader.count].name = EXTN_NAME_ERRHIST;
+                l_sens_list_idx = 0;
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = l_err_hist_idx;
+                l_sens_list_idx++;
+                l_extnSensorList[l_sensorHeader.count].data[l_sens_list_idx] = G_error_history[l_err_hist_idx];
+                l_sens_list_idx++;
+            }
+        }
+    }
+    if(l_sens_list_idx)
+    {
+        l_sensorHeader.count++;
+    }
+
+
     // Copy header to response buffer.
     memcpy ((void *) &(o_rsp_ptr->data[l_rsp_index]), (void *)&l_sensorHeader, sizeof(l_sensorHeader));
     //Increment index into response buffer.
diff --git a/src/occ_405/cmdh/cmdh_fsp_cmds.h b/src/occ_405/cmdh/cmdh_fsp_cmds.h
index 254c42a..3bd0f73 100755
--- a/src/occ_405/cmdh/cmdh_fsp_cmds.h
+++ b/src/occ_405/cmdh/cmdh_fsp_cmds.h
@@ -69,7 +69,9 @@ typedef enum
 #define EXTN_NAME_FNOM    0x464E4F4D // "FNOM"
 #define EXTN_NAME_FTURBO  0x46540000 // "FT"
 #define EXTN_NAME_FUTURBO 0x46555400 // "FUT"
+#define EXTN_NAME_ERRHIST 0x45525248 // "ERRH"
 
+#define MAX_EXTN_SENSORS 32
 //---------------------------------------------------------
 // Poll Command
 //---------------------------------------------------------
diff --git a/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c b/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
index d9b69f3..f285143 100755
--- a/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
+++ b/src/occ_405/cmdh/cmdh_fsp_cmds_datacnfg.c
@@ -1405,7 +1405,6 @@ errlHndl_t data_store_power_cap(const cmdh_fsp_cmd_t * i_cmd_ptr,
     cmdh_pcap_config_t * l_cmd_ptr = (cmdh_pcap_config_t *)i_cmd_ptr;
     uint16_t                        l_data_length = 0;
     uint32_t                        l_pcap_data_sz = 0;
-    static uint8_t                  L_pcap_count = 0;
     bool                            l_invalid_input = TRUE; //Assume bad input
 
     l_data_length = CONVERT_UINT8_ARRAY_UINT16(l_cmd_ptr->data_length[0], l_cmd_ptr->data_length[1]);
@@ -1475,8 +1474,7 @@ errlHndl_t data_store_power_cap(const cmdh_fsp_cmd_t * i_cmd_ptr,
         // It tells the master and slave code that there is new
         // pcap data.  This should not be incremented until
         // after the packet data has been copied into G_master_pcap_data.
-        L_pcap_count++;
-        G_master_pcap_data.pcap_data_count = L_pcap_count;
+        G_master_pcap_data.pcap_data_count++;
 
         // Change Data Request Mask to indicate we got the data
         // G_data_cnfg->data_mask |= DATA_MASK_PCAP_PRESENT;
diff --git a/src/occ_405/dcom/dcom.h b/src/occ_405/dcom/dcom.h
index 1bd69e1..109b4ef 100755
--- a/src/occ_405/dcom/dcom.h
+++ b/src/occ_405/dcom/dcom.h
@@ -99,6 +99,9 @@
 #define MAX_WAIT_FOR_SLAVES 400
 #define MAX_WAIT_FOR_MASTER 400
 
+// number of consecutive times a BCE request is not idle before tracing
+#define DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES 3
+
 // general defines
 #define TOD_SIZE 6
 #define NUM_TOD_SENSORS 3
diff --git a/src/occ_405/dcom/dcomMasterRx.c b/src/occ_405/dcom/dcomMasterRx.c
index f9565f1..2e26986 100644
--- a/src/occ_405/dcom/dcomMasterRx.c
+++ b/src/occ_405/dcom/dcomMasterRx.c
@@ -80,7 +80,8 @@ void setbit_slvoutbox_complete(uint8_t i_bit)
 // End Function Specification
 void task_dcom_rx_slv_outboxes( task_t *i_self)
 {
-    static uint32_t  L_wait4slaves = 0;
+    static uint32_t L_wait4slaves = 0;
+    static uint8_t  L_bce_not_ready_count = 0;
     uint32_t        l_orc = OCC_SUCCESS_REASON_CODE;
     uint32_t        l_orc_ext = OCC_NO_EXTENDED_RC;
     uint8_t         l_slv_response_mask = 0;
@@ -216,16 +217,20 @@ void task_dcom_rx_slv_outboxes( task_t *i_self)
                     // represents a hang condition that we can't recover from.
                     // DO NOT proceed with request create and schedule.
                     l_proceed_with_request_and_schedule = FALSE;
-                    // Trace important information from the request
-                    TRAC_INFO("BCE slv outbox rx request not idle and not complete, callback_rc[%d] options[0x%x] state[0x%x] abort_state[0x%x] completion_state[0x%x]",
-                              G_slv_outbox_rx_pba_request[l_slv].request.callback_rc,
-                              G_slv_outbox_rx_pba_request[l_slv].request.options,
-                              G_slv_outbox_rx_pba_request[l_slv].request.state,
-                              G_slv_outbox_rx_pba_request[l_slv].request.abort_state,
-                              G_slv_outbox_rx_pba_request[l_slv].request.completion_state);
 
-                    TRAC_INFO("NOT proceeding with BCE slv outbox rx request and schedule for slave[0x%02X]",
-                              l_slv);
+                    if(L_bce_not_ready_count == DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)
+                    {
+                        // Trace important information from the request
+                        TRAC_INFO("BCE slv outbox rx request not idle and not complete, callback_rc[%d] options[0x%x] state[0x%x] abort_state[0x%x] completion_state[0x%x]",
+                                  G_slv_outbox_rx_pba_request[l_slv].request.callback_rc,
+                                  G_slv_outbox_rx_pba_request[l_slv].request.options,
+                                  G_slv_outbox_rx_pba_request[l_slv].request.state,
+                                  G_slv_outbox_rx_pba_request[l_slv].request.abort_state,
+                                  G_slv_outbox_rx_pba_request[l_slv].request.completion_state);
+
+                        TRAC_INFO("NOT proceeding with BCE slv outbox rx request and schedule for slave[0x%02X]",
+                                  l_slv);
+                    }
                 }
                 else
                 {
@@ -235,6 +240,13 @@ void task_dcom_rx_slv_outboxes( task_t *i_self)
                 // Only proceed if the BCE request state checked out
                 if (l_proceed_with_request_and_schedule)
                 {
+                    if(L_bce_not_ready_count >= DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES) // previously not idle
+                    {
+                       TRAC_INFO("BCE slv outbox rx request idle and complete after %d times", L_bce_not_ready_count);
+                    }
+
+                    L_bce_not_ready_count = 0;
+
                     // Copy request from main memory to SRAM
                     l_ssxrc = bce_request_create(
                                     &G_slv_outbox_rx_pba_request[l_slv],             // block copy object
@@ -283,6 +295,11 @@ void task_dcom_rx_slv_outboxes( task_t *i_self)
                         break;
                     }
                 }
+                else
+                {
+                    L_bce_not_ready_count++;
+                    INCREMENT_ERR_HISTORY(ERR_DCOM_RX_SLV_OUTBOX);
+                }
             }
             else
             {
@@ -366,7 +383,8 @@ uint32_t dcom_rx_slv_outbox_doorbell( void )
 
     if (l_pbarc != 0)
     {
-        // Failure occurred but only trace it once
+        INCREMENT_ERR_HISTORY(ERR_DCOM_MASTER_PBAX_READ_FAIL);
+
         TRAC_ERR("Master PBAX Read Failure in receiving unicast slave doorbells - RC[%08X]", l_pbarc);
 
         // Handle pbax read failure on queue 1
diff --git a/src/occ_405/dcom/dcomMasterTx.c b/src/occ_405/dcom/dcomMasterTx.c
index 53b7b33..e1a46bc 100644
--- a/src/occ_405/dcom/dcomMasterTx.c
+++ b/src/occ_405/dcom/dcomMasterTx.c
@@ -89,6 +89,8 @@ uint32_t dcom_build_slv_inbox(void)
     // interrupt context.
     if(G_pbax_rc)
     {
+        INCREMENT_ERR_HISTORY(ERR_DCOM_MASTER_PBAX_SEND_FAIL);
+
         if (!L_traced)
         {
             TRAC_INFO("PBAX Send Failure in transimitting multicast doorbell - RC[%08X], packet[%d]", G_pbax_rc, G_pbax_packet);
@@ -225,6 +227,7 @@ uint32_t dcom_which_buffer(void)
 void task_dcom_tx_slv_inbox( task_t *i_self)
 {
     static bool L_error = FALSE;
+    static uint8_t L_bce_not_ready_count = 0;
     uint32_t    l_orc = OCC_SUCCESS_REASON_CODE;
     uint32_t    l_orc_ext = OCC_NO_EXTENDED_RC;
     uint64_t    l_start = ssx_timebase_get();
@@ -320,14 +323,17 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
                 // DO NOT proceed with request create and schedule.
                 l_proceed_with_request_and_schedule = FALSE;
 
-                // Trace important information from the request
-                TRAC_INFO("BCE slv inbox tx request not idle and not complete: callback_rc[%d] options[0x%x] state[0x%x] abort_state[0x%x] completion_state[0x%x]",
-                          G_slv_inbox_tx_pba_request.request.callback_rc,
-                          G_slv_inbox_tx_pba_request.request.options,
-                          G_slv_inbox_tx_pba_request.request.state,
-                          G_slv_inbox_tx_pba_request.request.abort_state,
-                          G_slv_inbox_tx_pba_request.request.completion_state);
-                TRAC_INFO("NOT proceeding with BCE slv inbox tx request and schedule");
+                if(L_bce_not_ready_count == DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)
+                {
+                   // Trace important information from the request
+                   TRAC_INFO("BCE slv inbox tx request not idle and not complete: callback_rc[%d] options[0x%x] state[0x%x] abort_state[0x%x] completion_state[0x%x]",
+                             G_slv_inbox_tx_pba_request.request.callback_rc,
+                             G_slv_inbox_tx_pba_request.request.options,
+                             G_slv_inbox_tx_pba_request.request.state,
+                             G_slv_inbox_tx_pba_request.request.abort_state,
+                             G_slv_inbox_tx_pba_request.request.completion_state);
+                   TRAC_INFO("NOT proceeding with BCE slv inbox tx request and schedule");
+                }
             }
             else
             {
@@ -337,6 +343,13 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
             // Only proceed if the BCE request state checked out
             if (l_proceed_with_request_and_schedule)
             {
+                if(L_bce_not_ready_count >= DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)  // previously not idle
+                {
+                   TRAC_INFO("BCE slv inbox tx request idle and complete after %d times", L_bce_not_ready_count);
+                }
+
+                L_bce_not_ready_count = 0;
+
                 // Set up inboxes copy request
                 l_ssxrc = bce_request_create(
                                 &G_slv_inbox_tx_pba_request,        // Block copy object
@@ -389,6 +402,12 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
                     break;
                 }
             }
+            else
+            {
+                L_bce_not_ready_count++;
+                INCREMENT_ERR_HISTORY(ERR_DCOM_TX_SLV_INBOX);
+            }
+
             // Moved the break statement here in case we decide not to
             // schedule the BCE request.
             break;
@@ -402,6 +421,8 @@ void task_dcom_tx_slv_inbox( task_t *i_self)
             }
             else
             {
+                INCREMENT_ERR_HISTORY(ERR_DCOM_APSS_COMPLETE_TIMEOUT);
+
                 //Failure occurred, step up the FAIL_COUNT
                 APSS_FAIL();
 
diff --git a/src/occ_405/dcom/dcomSlaveRx.c b/src/occ_405/dcom/dcomSlaveRx.c
index 43c9829..a4b6e2d 100644
--- a/src/occ_405/dcom/dcomSlaveRx.c
+++ b/src/occ_405/dcom/dcomSlaveRx.c
@@ -95,6 +95,7 @@ void dcom_rx_slv_inbox_callback( void )
 // End Function Specification
 void task_dcom_rx_slv_inbox( task_t *i_self)
 {
+    static uint8_t L_bce_not_ready_count = 0;
     uint32_t    l_orc = OCC_SUCCESS_REASON_CODE;
     uint32_t    l_orc_ext = OCC_NO_EXTENDED_RC;
     uint64_t    l_start = ssx_timebase_get();
@@ -195,16 +196,19 @@ void task_dcom_rx_slv_inbox( task_t *i_self)
                     // DO NOT proceed with request create and schedule.
                     l_proceed_with_request_and_schedule = FALSE;
 
-                    // Trace important information from the request
-                    TRAC_INFO("BCE slv inbox rx request not idle and not complete, \
-                              callback_rc=%d options=0x%x state=0x%x abort_state=0x%x \
-                              completion_state=0x%x",
-                              G_slv_inbox_rx_pba_request.request.callback_rc,
-                              G_slv_inbox_rx_pba_request.request.options,
-                              G_slv_inbox_rx_pba_request.request.state,
-                              G_slv_inbox_rx_pba_request.request.abort_state,
-                              G_slv_inbox_rx_pba_request.request.completion_state);
-                    TRAC_INFO("NOT proceeding with BCE slv inbox rx request and schedule");
+                    if(L_bce_not_ready_count == DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)
+                    {
+                        // Trace important information from the request
+                        TRAC_INFO("BCE slv inbox rx request not idle and not complete, \
+                                  callback_rc=%d options=0x%x state=0x%x abort_state=0x%x \
+                                  completion_state=0x%x",
+                                  G_slv_inbox_rx_pba_request.request.callback_rc,
+                                  G_slv_inbox_rx_pba_request.request.options,
+                                  G_slv_inbox_rx_pba_request.request.state,
+                                  G_slv_inbox_rx_pba_request.request.abort_state,
+                                  G_slv_inbox_rx_pba_request.request.completion_state);
+                        TRAC_INFO("NOT proceeding with BCE slv inbox rx request and schedule");
+                    }
                 }
                 else
                 {
@@ -214,6 +218,13 @@ void task_dcom_rx_slv_inbox( task_t *i_self)
                 // Only proceed if the BCE request state checked out
                 if (l_proceed_with_request_and_schedule)
                 {
+                    if(L_bce_not_ready_count >= DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)  // previously not idle
+                    {
+                       TRAC_INFO("BCE slv inbox rx request idle and complete after %d times", L_bce_not_ready_count);
+                    }
+
+                    L_bce_not_ready_count = 0;
+
                     // Copy request from main memory to SRAM
                     l_ssxrc = bce_request_create(
                                     &G_slv_inbox_rx_pba_request,        // Block copy object
@@ -264,6 +275,11 @@ void task_dcom_rx_slv_inbox( task_t *i_self)
                     }
                     break;
                 }
+                else
+                {
+                    L_bce_not_ready_count++;
+                    INCREMENT_ERR_HISTORY(ERR_DCOM_RX_SLV_INBOX);
+                }
             }
             else
             {
@@ -356,6 +372,7 @@ uint32_t dcom_rx_slv_inbox_doorbell( void )
         // We got an error reading from the PBAX, return to caller
         if ( l_pbarc != 0 )
         {
+            INCREMENT_ERR_HISTORY(ERR_DCOM_SLAVE_PBAX_READ_FAIL);
             G_dcomTime.slave.doorbellErrorFlags |= DCOM_DOORBELL_HW_ERR;
             // Failure occurred
             TRAC_ERR("Slave PBAX Read Failure in receiving multicast doorbell from master - RC[%08X]", l_pbarc);
@@ -461,6 +478,7 @@ void task_dcom_wait_for_master( task_t *i_self)
     uint32_t            l_num_read              = 0;
     static bool         L_first_doorbell_rcvd   = FALSE;
     static bool         L_queue_enabled         = FALSE;
+    static bool         L_Pmax_error_logged     = FALSE;
     static uint32_t     L_pobid_retries_left    = POBID_RETRIES;
     static uint16_t     L_no_master_doorbell_cnt = 0;
     static uint16_t     L_trace_every_count = 1;
@@ -504,48 +522,52 @@ void task_dcom_wait_for_master( task_t *i_self)
                     G_apss_lower_pmax_rail = TRUE;
 
                     // Create and commit this error only once
-                    TRAC_ERR("Detected a problem with slave data collection: soft time-out[%d]. Lowering Pmax_rail!",
-                             APSS_DATA_FAIL_PMAX_RAIL);
-
-                    /* @
-                     * @errortype
-                     * @moduleid    DCOM_MID_TASK_WAIT_FOR_MASTER
-                     * @reasoncode  APSS_SLV_SHORT_TIMEOUT
-                     * @userdata1   Time-out value
-                     * @userdata4   OCC_NO_EXTENDED_RC
-                     * @devdesc     Detected a problem with APSS data collection (short time-out)
-                     */
-                    errlHndl_t  l_errl = createErrl(
-                                                    DCOM_MID_TASK_WAIT_FOR_MASTER,  //modId
-                                                    APSS_SLV_SHORT_TIMEOUT,         //reasoncode
-                                                    OCC_NO_EXTENDED_RC,             //Extended reason code
-                                                    ERRL_SEV_INFORMATIONAL,         //Severity
-                                                    NULL,                           //Trace Buf
-                                                    DEFAULT_TRACE_SIZE,             //Trace Size
-                                                    APSS_DATA_FAIL_PMAX_RAIL,       //userdata1
-                                                    0                               //userdata2
-                                                   );
-
-                    // Callout to firmware
-                    addCalloutToErrl(l_errl,
-                                     ERRL_CALLOUT_TYPE_COMPONENT_ID,
-                                     ERRL_COMPONENT_ID_FIRMWARE,
-                                     ERRL_CALLOUT_PRIORITY_MED);
-
-                    // Callout to processor
-                    addCalloutToErrl(l_errl,
-                                     ERRL_CALLOUT_TYPE_HUID,
-                                     G_sysConfigData.proc_huid,
-                                     ERRL_CALLOUT_PRIORITY_LOW);
-
-                    // Callout to APSS
-                    addCalloutToErrl(l_errl,
-                                     ERRL_CALLOUT_TYPE_HUID,
-                                     G_sysConfigData.apss_huid,
-                                     ERRL_CALLOUT_PRIORITY_LOW);
-
-                    setErrlActions(l_errl, ERRL_ACTIONS_MANUFACTURING_ERROR);
-                    commitErrl(&l_errl);
+                    if(!L_Pmax_error_logged)
+                    {
+                       TRAC_ERR("Detected a problem with slave data collection: soft time-out[%d]. Lowering Pmax_rail!",
+                                APSS_DATA_FAIL_PMAX_RAIL);
+
+                       /* @
+                        * @errortype
+                        * @moduleid    DCOM_MID_TASK_WAIT_FOR_MASTER
+                        * @reasoncode  APSS_SLV_SHORT_TIMEOUT
+                        * @userdata1   Time-out value
+                        * @userdata4   OCC_NO_EXTENDED_RC
+                        * @devdesc     Detected a problem with APSS data collection (short time-out)
+                        */
+                       errlHndl_t  l_errl = createErrl(
+                                                       DCOM_MID_TASK_WAIT_FOR_MASTER,  //modId
+                                                       APSS_SLV_SHORT_TIMEOUT,         //reasoncode
+                                                       OCC_NO_EXTENDED_RC,             //Extended reason code
+                                                       ERRL_SEV_INFORMATIONAL,         //Severity
+                                                       NULL,                           //Trace Buf
+                                                       DEFAULT_TRACE_SIZE,             //Trace Size
+                                                       APSS_DATA_FAIL_PMAX_RAIL,       //userdata1
+                                                       0                               //userdata2
+                                                      );
+
+                       // Callout to firmware
+                       addCalloutToErrl(l_errl,
+                                        ERRL_CALLOUT_TYPE_COMPONENT_ID,
+                                        ERRL_COMPONENT_ID_FIRMWARE,
+                                        ERRL_CALLOUT_PRIORITY_MED);
+
+                       // Callout to processor
+                       addCalloutToErrl(l_errl,
+                                        ERRL_CALLOUT_TYPE_HUID,
+                                        G_sysConfigData.proc_huid,
+                                        ERRL_CALLOUT_PRIORITY_LOW);
+
+                       // Callout to APSS
+                       addCalloutToErrl(l_errl,
+                                        ERRL_CALLOUT_TYPE_HUID,
+                                        G_sysConfigData.apss_huid,
+                                        ERRL_CALLOUT_PRIORITY_LOW);
+
+                       setErrlActions(l_errl, ERRL_ACTIONS_MANUFACTURING_ERROR);
+                       commitErrl(&l_errl);
+                       L_Pmax_error_logged = TRUE;
+                    }
                 }
 
                 if (L_no_master_doorbell_cnt == APSS_DATA_FAIL_MAX)
diff --git a/src/occ_405/dcom/dcomSlaveTx.c b/src/occ_405/dcom/dcomSlaveTx.c
index a7060cb..33ef166 100755
--- a/src/occ_405/dcom/dcomSlaveTx.c
+++ b/src/occ_405/dcom/dcomSlaveTx.c
@@ -146,12 +146,13 @@ uint32_t dcom_calc_slv_outbox_addr( const dcom_slv_outbox_doorbell_t * i_doorbel
 // End Function Specification
 void task_dcom_tx_slv_outbox( task_t *i_self)
 {
-    static bool l_error = FALSE;
-    uint32_t    l_orc = OCC_SUCCESS_REASON_CODE;
-    uint32_t    l_orc_ext = OCC_NO_EXTENDED_RC;
+    static bool    L_error = FALSE;
+    static uint8_t L_bce_not_ready_count = 0;
+    uint32_t       l_orc = OCC_SUCCESS_REASON_CODE;
+    uint32_t       l_orc_ext = OCC_NO_EXTENDED_RC;
     // Use a static local bool to track whether the BCE request used
     // here has ever been successfully created at least once
-    static bool L_bce_slv_outbox_tx_request_created_once = FALSE;
+    static bool    L_bce_slv_outbox_tx_request_created_once = FALSE;
 
     DCOM_DBG("3. TX Slave Outboxes\n");
 
@@ -162,6 +163,7 @@ void task_dcom_tx_slv_outbox( task_t *i_self)
         if(G_slave_pbax_rc)
         {
             TRAC_ERR("task_dcom_tx_slv_outbox: PBAX Send Failure in transimitting doorbell - RC[%08X]", G_slave_pbax_rc);
+            INCREMENT_ERR_HISTORY(ERR_DCOM_SLAVE_PBAX_SEND_FAIL);
         }
 
         // Build/setup outbox
@@ -221,16 +223,19 @@ void task_dcom_tx_slv_outbox( task_t *i_self)
             // DO NOT proceed with request create and schedule.
             l_proceed_with_request_and_schedule = FALSE;
 
-            // Trace important information from the request
-            TRAC_INFO("BCE slv outbox tx request not idle and not complete, \
-                      callback_rc=%d options=0x%x state=0x%x abort_state=0x%x \
-                      completion_state=0x%x",
-                      G_slv_outbox_tx_pba_request.request.callback_rc,
-                      G_slv_outbox_tx_pba_request.request.options,
-                      G_slv_outbox_tx_pba_request.request.state,
-                      G_slv_outbox_tx_pba_request.request.abort_state,
-                      G_slv_outbox_tx_pba_request.request.completion_state);
-            TRAC_INFO("NOT proceeding with BCE slv outbox tx request and schedule");
+            if(L_bce_not_ready_count == DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)
+            {
+                // Trace important information from the request
+                TRAC_INFO("BCE slv outbox tx request not idle and not complete, \
+                          callback_rc=%d options=0x%x state=0x%x abort_state=0x%x \
+                          completion_state=0x%x",
+                          G_slv_outbox_tx_pba_request.request.callback_rc,
+                          G_slv_outbox_tx_pba_request.request.options,
+                          G_slv_outbox_tx_pba_request.request.state,
+                          G_slv_outbox_tx_pba_request.request.abort_state,
+                          G_slv_outbox_tx_pba_request.request.completion_state);
+                TRAC_INFO("NOT proceeding with BCE slv outbox tx request and schedule");
+            }
         }
         else
         {
@@ -240,6 +245,13 @@ void task_dcom_tx_slv_outbox( task_t *i_self)
         // Only proceed if the BCE request state checked out
         if (l_proceed_with_request_and_schedule)
         {
+            if(L_bce_not_ready_count >= DCOM_TRACE_NOT_IDLE_AFTER_CONSEC_TIMES)  // previously not idle
+            {
+               TRAC_INFO("BCE slv outbox tx request idle and complete after %d times", L_bce_not_ready_count);
+            }
+
+            L_bce_not_ready_count = 0;
+
             // set up outbox copy request
             l_ssxrc = bce_request_create(
                             &G_slv_outbox_tx_pba_request,       // Block copy object
@@ -289,10 +301,15 @@ void task_dcom_tx_slv_outbox( task_t *i_self)
                 break;
             }
         }
+        else
+        {
+            L_bce_not_ready_count++;
+            INCREMENT_ERR_HISTORY(ERR_DCOM_TX_SLV_OUTBOX);
+        }
 
     } while (0);
 
-    if ( l_orc != OCC_SUCCESS_REASON_CODE && l_error == FALSE)
+    if ( l_orc != OCC_SUCCESS_REASON_CODE && L_error == FALSE)
     {
         // Create and commit error
         // See return code doxygen tags for error description
@@ -316,7 +333,7 @@ void task_dcom_tx_slv_outbox( task_t *i_self)
         // request a reset
         REQUEST_RESET( l_errl );
 
-        l_error = TRUE;
+        L_error = TRUE;
 
     }
 
diff --git a/src/occ_405/dcom/dcom_thread.c b/src/occ_405/dcom/dcom_thread.c
index ada5bfc..2b9dcb7 100755
--- a/src/occ_405/dcom/dcom_thread.c
+++ b/src/occ_405/dcom/dcom_thread.c
@@ -94,14 +94,17 @@ void Dcom_thread_routine(void *arg)
         // --------------------------------------------------
         G_dcom_thread_counter++;
 
-// NOTE: Temporary system config must say we are FSP system so
-// that we don't try to access main memory here.
         // --------------------------------------------------
         // Check if we need to update the opal table
+        // only start checking after OCC has gone thru state change
         // --------------------------------------------------
-        if(G_sysConfigData.system_type.kvm)
+        if( (CURRENT_STATE() >= OCC_STATE_OBSERVATION) || (isSafeStateRequested()) )
         {
-            check_for_opal_updates();
+            // stop checking if we hit a critical error trying to update memory
+            if(G_opal_table_update_state != OPAL_TABLE_UPDATE_CRITICAL_ERROR)
+            {
+                check_for_opal_updates();
+            }
         }
 
         // --------------------------------------------------
diff --git a/src/occ_405/errl/errl.h b/src/occ_405/errl/errl.h
index 6e8b589..5518369 100755
--- a/src/occ_405/errl/errl.h
+++ b/src/occ_405/errl/errl.h
@@ -280,6 +280,16 @@ typedef enum {
     ERR_AVSBUS_VDN_OVER_TEMPERATURE =  8,
     ERR_AVSBUS_VDD_OVER_CURRENT     =  9,
     ERR_AVSBUS_VDN_OVER_CURRENT     = 10,
+    ERR_INVALID_APSS_DATA           = 11,
+    ERR_DCOM_APSS_COMPLETE_TIMEOUT  = 12,
+    ERR_DCOM_TX_SLV_INBOX           = 13,
+    ERR_DCOM_RX_SLV_INBOX           = 14,
+    ERR_DCOM_TX_SLV_OUTBOX          = 15,
+    ERR_DCOM_RX_SLV_OUTBOX          = 16,
+    ERR_DCOM_MASTER_PBAX_SEND_FAIL  = 17,
+    ERR_DCOM_SLAVE_PBAX_SEND_FAIL   = 18,
+    ERR_DCOM_MASTER_PBAX_READ_FAIL  = 19,
+    ERR_DCOM_SLAVE_PBAX_READ_FAIL   = 20,
 } ERR_HISTORY_INDEX;
 #define INCREMENT_ERR_HISTORY(errorIndex) { \
     if ((errorIndex < ERR_HISTORY_SIZE) && (G_error_history[errorIndex] < 255)) { \
diff --git a/src/occ_405/occ_service_codes.h b/src/occ_405/occ_service_codes.h
index 2b066df..efdb838 100644
--- a/src/occ_405/occ_service_codes.h
+++ b/src/occ_405/occ_service_codes.h
@@ -143,6 +143,7 @@ enum occReasonCode
     SENSOR_MAIN_MEM_ERROR           = 0xF0,
 
     INBAND_CMD_ERROR                = 0xF1,
+    OPAL_TABLE_UPDATE_ERROR         = 0xF2,
 
     /// Success!
     OCC_SUCCESS_REASON_CODE         = 0xFF,
diff --git a/src/occ_405/proc/proc_data_service_codes.h b/src/occ_405/proc/proc_data_service_codes.h
index 7e0ab19..40fc0df 100755
--- a/src/occ_405/proc/proc_data_service_codes.h
+++ b/src/occ_405/proc/proc_data_service_codes.h
@@ -39,6 +39,8 @@ enum procModuleId
     PROC_TASK_NEST_DTS_MOD          = PROC_COMP_ID | 0x06,
     PROC_NEST_DTS_INIT_MOD          = PROC_COMP_ID | 0x07,
     PROC_24X7_MOD                   = PROC_COMP_ID | 0x08,
+    PROC_POP_OPAL_TBL_TO_MEM_MOD    = PROC_COMP_ID | 0x09,
+    PROC_CHECK_FOR_OPAL_UPDATES_MOD = PROC_COMP_ID | 0x0A,
 };
 
 #endif /* #ifndef _PROC_DATA_SERVICE_CODES_H_ */
diff --git a/src/occ_405/proc/proc_pstate.c b/src/occ_405/proc/proc_pstate.c
index 277399d..b80e57b 100755
--- a/src/occ_405/proc/proc_pstate.c
+++ b/src/occ_405/proc/proc_pstate.c
@@ -66,12 +66,20 @@ pstateStatus G_proc_pstate_status = PSTATES_DISABLED;
 PMCR_OWNER G_proc_pmcr_owner = PMCR_OWNER_HOST;
 
 // OPAL Dynamic data, updated whenever any OCC G_opal_table.dynamic parameter change
+// Since this is happening multiple times need to keep track of it being scheduled
 DMA_BUFFER( opal_dynamic_table_t G_opal_dynamic_table ) = {{0}};
+bool G_opal_dynamic_bce_req_scheduled = false;
+BceRequest G_opal_dynamic_bce_req;
+#define OPAL_DYNAMIC_UPDATE_BCE_RETRIES 2
 
 // OPAL Static data, updated once at transition to active state
 DMA_BUFFER( opal_static_table_t  G_opal_static_table )  = {{0}};
+BceRequest G_opal_static_bce_req;
 
 
+
+volatile uint8_t G_opal_table_update_state = OPAL_TABLE_UPDATE_IDLE;
+
 // Function Specification
 //
 // Name:  proc_is_hwpstate_enabled
@@ -157,30 +165,36 @@ Pstate proc_freq2pstate(uint32_t i_freq_mhz)
 //
 // Name:  proc_pstate_kvm_setup
 //
-// Description: Get everything set up for KVM mode
+// Description: Copy Pstate table to OPAL shared memory this should only be called once
+//    when going to active state
 //
 // End Function Specification
 void proc_pstate_kvm_setup()
 {
-    do
-    {
-        //only run this in KVM mode
-        if(!G_sysConfigData.system_type.kvm)
-        {
-            TRAC_ERR("proc_pstate_kvm_setup: called in a non OPAL system");
-            break;
-        }
-
-        TRAC_INFO("proc_pstate_kvm_setup: populate static OPAL data");
+    TRAC_IMP("proc_pstate_kvm_setup: populate static OPAL data");
 
-        // Initialize the opal table in SRAM (sets valid bit)
-        populate_opal_static_data();
+    // Initialize the opal table in SRAM (sets valid bit)
+    populate_opal_static_data();
 
-        // copy sram image into mainstore HOMER
-        populate_opal_tbl_to_mem(OPAL_STATIC);
-        TRAC_IMP("proc_pstate_kvm_setup: RUNNING IN KVM MODE");
+    // copy sram image into mainstore HOMER
+    populate_opal_tbl_to_mem(OPAL_STATIC);
+}
 
-    }while(0);
+// Function Specification
+//
+// Name:  opal_table_bce_callback
+//
+// Description: Callback function for populate_opal_tbl_to_mem() BCE request
+//              NO TRACING OR CALLING FUNCTIONS THAT TRACE ALLOWED
+//
+// End Function Specification
+void opal_table_bce_callback( void )
+{
+    // If the BCE that just finished was for a dynamic table update notify host
+    if(G_opal_table_update_state == OPAL_TABLE_UPDATE_DYNAMIC_COPY)
+    {
+        G_opal_table_update_state = OPAL_TABLE_UPDATE_NOTIFY_HOST;
+    }
 }
 
 // Function Specification
@@ -210,10 +224,11 @@ void populate_opal_dynamic_data()
     G_opal_dynamic_table.dynamic.mem_throt_status     = G_amec_opal_mem_throt_reason;
     G_opal_dynamic_table.dynamic.quick_power_drop     = AMEC_INTF_GET_OVERSUBSCRIPTION();
     G_opal_dynamic_table.dynamic.power_shift_ratio    = G_sysConfigData.psr;
-    G_opal_dynamic_table.dynamic.power_cap_type       = G_master_pcap_data.source;
-    G_opal_dynamic_table.dynamic.min_power_cap        = G_master_pcap_data.soft_min_pcap;
-    G_opal_dynamic_table.dynamic.max_power_cap        = G_master_pcap_data.max_pcap;
-    G_opal_dynamic_table.dynamic.current_power_cap    = G_master_pcap_data.current_pcap;
+    G_opal_dynamic_table.dynamic.power_cap_type       = G_sysConfigData.pcap.source;
+    G_opal_dynamic_table.dynamic.min_power_cap        = G_sysConfigData.pcap.hard_min_pcap;
+    G_opal_dynamic_table.dynamic.max_power_cap        = G_sysConfigData.pcap.max_pcap;
+    G_opal_dynamic_table.dynamic.current_power_cap    = g_amec->pcap.active_node_pcap;
+    G_opal_dynamic_table.dynamic.soft_min_power_cap   = G_sysConfigData.pcap.soft_min_pcap;
 }
 
 // Function Specification
@@ -294,21 +309,33 @@ void populate_opal_tbl_to_mem(opalDataType opal_data_type)
     uint32_t l_reasonCode = 0;
     uint32_t l_extReasonCode = 0;
 
-    uint32_t mainstore_address;
-    uint32_t sram_address;
-    size_t block_size;
-
+    // Set up copy request for type of data being updated.  NOTE:  only DYNAMIC uses the callback
     if(opal_data_type == OPAL_STATIC)
     {
-        mainstore_address = OPAL_STATIC_ADDRESS_HOMER;
-        sram_address      = (uint32_t) &G_opal_static_table;
-        block_size        = (size_t) sizeof(G_opal_static_table);
+        // static is only updated when going active and should be blocking to make sure
+        // this complets before reporting back that the state change finished
+        l_ssxrc = bce_request_create(&G_opal_static_bce_req,                // block copy object
+                                     &G_pba_bcue_queue,                     // sram to mainstore copy engine
+                                     OPAL_STATIC_ADDRESS_HOMER,             // mainstore address
+                                     (uint32_t) &G_opal_static_table,       // sram starting address
+                                     (size_t) sizeof(G_opal_static_table),  // size of copy
+                                     SSX_SECONDS(2),                        // timeout
+                                     NULL,                                  // no call back
+                                     NULL,                                  // call back arguments
+                                     ASYNC_REQUEST_BLOCKING);
     }
     else if(opal_data_type == OPAL_DYNAMIC)
     {
-        mainstore_address = OPAL_DYNAMIC_ADDRESS_HOMER;
-        sram_address      =(uint32_t) &G_opal_dynamic_table;
-        block_size        = (size_t) sizeof(G_opal_dynamic_table);
+        // dynamic data can be updated while active and should NOT be blocking
+        l_ssxrc = bce_request_create(&G_opal_dynamic_bce_req,               // block copy object
+                                     &G_pba_bcue_queue,                     // sram to mainstore copy engine
+                                     OPAL_DYNAMIC_ADDRESS_HOMER,            // mainstore address
+                                     (uint32_t) &G_opal_dynamic_table,      // sram starting address
+                                     (size_t) sizeof(G_opal_dynamic_table), // size of copy
+                                     SSX_WAIT_FOREVER,                      // no timeout
+              (AsyncRequestCallback) opal_table_bce_callback,               // call back
+                                     NULL,                                  // call back arguments
+                                     ASYNC_CALLBACK_IMMEDIATE);
     }
     else
     {
@@ -318,39 +345,32 @@ void populate_opal_tbl_to_mem(opalDataType opal_data_type)
 
     do
     {
-        BceRequest pba_copy;
-        // Set up copy request
-        l_ssxrc = bce_request_create(
-                       &pba_copy,                   // block copy object
-                       &G_pba_bcue_queue,           // sram to mainstore copy engine
-                       mainstore_address,           // mainstore address
-                       sram_address,                // sram starting address
-                       block_size,                  // size of copy
-                       SSX_WAIT_FOREVER,            // no timeout
-                       NULL,                        // call back
-                       NULL,                        // call back arguments
-                       ASYNC_REQUEST_BLOCKING       // callback mask
-                       );
-
         if(l_ssxrc != SSX_OK)
         {
             TRAC_ERR("populate_opal_tbl_to_mem: PBA request create failure rc=[%08X]",
                      -l_ssxrc);
             /*
              * @errortype
-             * @moduleid    MAIN_STATE_TRANSITION_MID
-             * @reasoncode  SSX_GENERIC_FAILURE
+             * @moduleid    PROC_POP_OPAL_TBL_TO_MEM_MOD
+             * @reasoncode  OPAL_TABLE_UPDATE_ERROR
              * @userdata1   RC for PBA block-copy engine
              * @userdata4   ERC_BCE_REQUEST_CREATE_FAILURE
              * @devdesc     Failed to create BCUE request
              */
-            l_reasonCode = SSX_GENERIC_FAILURE;
+            l_reasonCode = OPAL_TABLE_UPDATE_ERROR;
             l_extReasonCode = ERC_BCE_REQUEST_CREATE_FAILURE;
             break;
         }
 
         // Do actual copying
-        l_ssxrc = bce_request_schedule(&pba_copy);
+        if(opal_data_type == OPAL_STATIC)
+        {
+            l_ssxrc = bce_request_schedule(&G_opal_static_bce_req);
+        }
+        else
+        {
+            l_ssxrc = bce_request_schedule(&G_opal_dynamic_bce_req);
+        }
 
         if(l_ssxrc != SSX_OK)
         {
@@ -358,13 +378,13 @@ void populate_opal_tbl_to_mem(opalDataType opal_data_type)
                      -l_ssxrc);
             /*
              * @errortype
-             * @moduleid    MAIN_STATE_TRANSITION_MID
-             * @reasoncode  SSX_GENERIC_FAILURE
+             * @moduleid    PROC_POP_OPAL_TBL_TO_MEM_MOD
+             * @reasoncode  OPAL_TABLE_UPDATE_ERROR
              * @userdata1   RC for PBA block-copy engine
              * @userdata4   ERC_BCE_REQUEST_SCHEDULE_FAILURE
              * @devdesc     Failed to copy OPAL data by using BCUE
              */
-            l_reasonCode = SSX_GENERIC_FAILURE;
+            l_reasonCode = OPAL_TABLE_UPDATE_ERROR;
             l_extReasonCode = ERC_BCE_REQUEST_SCHEDULE_FAILURE;
             break;
         }
@@ -372,22 +392,43 @@ void populate_opal_tbl_to_mem(opalDataType opal_data_type)
 
     if ( l_ssxrc != SSX_OK )
     {
-        errlHndl_t l_errl = createErrl(MAIN_STATE_TRANSITION_MID,  //modId
-                                       l_reasonCode,               //reasoncode
-                                       l_extReasonCode,            //Extended reason code
-                                       ERRL_SEV_UNRECOVERABLE,     //Severity
-                                       NULL,                       //Trace Buf
-                                       0,                          //Trace Size
-                                       -l_ssxrc,                   //userdata1
-                                       0);                         //userdata2
-
-        // Callout firmware
-        addCalloutToErrl(l_errl,
-                         ERRL_CALLOUT_TYPE_COMPONENT_ID,
-                         ERRL_COMPONENT_ID_FIRMWARE,
-                         ERRL_CALLOUT_PRIORITY_HIGH);
-
-        commitErrl(&l_errl);
+        // set back to idle since callback won't happen
+        if( (opal_data_type == OPAL_DYNAMIC) &&
+            (G_opal_table_update_state == OPAL_TABLE_UPDATE_DYNAMIC_COPY) )
+        {
+            G_opal_table_update_state = OPAL_TABLE_UPDATE_IDLE;
+        }
+
+        // data in main mem only matters for OPAL so only log error if OPAL
+        if(G_sysConfigData.system_type.kvm)
+        {
+            errlHndl_t l_errl = createErrl(PROC_POP_OPAL_TBL_TO_MEM_MOD,  //modId
+                                           l_reasonCode,                  //reasoncode
+                                           l_extReasonCode,               //Extended reason code
+                                           ERRL_SEV_UNRECOVERABLE,        //Severity
+                                           NULL,                          //Trace Buf
+                                           0,                             //Trace Size
+                                           -l_ssxrc,                      //userdata1
+                                           0);                            //userdata2
+
+            // Callout firmware
+            addCalloutToErrl(l_errl,
+                             ERRL_CALLOUT_TYPE_COMPONENT_ID,
+                             ERRL_COMPONENT_ID_FIRMWARE,
+                             ERRL_CALLOUT_PRIORITY_HIGH);
+
+            // Callout processor
+            addCalloutToErrl(l_errl,
+                             ERRL_CALLOUT_TYPE_HUID,
+                             G_sysConfigData.proc_huid,
+                             ERRL_CALLOUT_PRIORITY_MED);
+
+            commitErrl(&l_errl);
+        }
+    }
+    else if(opal_data_type == OPAL_DYNAMIC)
+    {
+        G_opal_dynamic_bce_req_scheduled = true;
     }
 }
 
@@ -396,52 +437,177 @@ void populate_opal_tbl_to_mem(opalDataType opal_data_type)
 //
 // Name: check_for_opal_updates
 //
-// Description: Checks if the opal table needs an update
-//              and updates if necessary.
+// Description: Checks if any of the dynamic fields in the opal shared memory
+//              needs to be updated and updates if necessary.
 //
 // End Function Specification
 void check_for_opal_updates(void)
 {
-    bool throttle_change = false;
+    bool           dynamic_data_change = false;
+    bool           l_log_crit_error = false;
+    static uint8_t L_num_bce_checks = 0;
 
-    // if throttle status change, update OPAL and notify host
-    if( (G_opal_dynamic_table.dynamic.proc_throt_status != G_amec_opal_proc_throt_reason) ||         // PROC throttle states changed
-        ((G_opal_dynamic_table.dynamic.proc_throt_status != OCC_RESET) && isSafeStateRequested()) ||  // OCC reset requested & OPAL not updated
-        (G_opal_dynamic_table.dynamic.mem_throt_status  != G_amec_opal_mem_throt_reason) )           // Mem throttle status changed
+    // check if BCE for previous change finished and now need to notify host
+    if(G_opal_table_update_state == OPAL_TABLE_UPDATE_NOTIFY_HOST)
     {
-        throttle_change = true;
-        TRAC_INFO("check_for_opal_updates: throttle status change - proc 0x%02X->0x%02X, mem: 0x%02X->0x%02X",
-                  G_opal_dynamic_table.dynamic.proc_throt_status, G_amec_opal_proc_throt_reason,
-                  G_opal_dynamic_table.dynamic.mem_throt_status, G_amec_opal_mem_throt_reason);
-        update_dynamic_opal_data();
+         // regardless of if we notify host we are done with this change
+         G_opal_table_update_state = OPAL_TABLE_UPDATE_IDLE;
+
+         if(G_sysConfigData.system_type.kvm)  // only notify if OPAL
+         {
+              notify_host(INTR_REASON_OPAL_SHARED_MEM_CHANGE);
+         }
     }
 
-    // else, if a dynamic OPAL parameter changed, update OPAL dynamic table and OPAL data in memory
-    else if(G_opal_dynamic_table.dynamic.occ_state         != CURRENT_STATE()                  ||
-            G_opal_dynamic_table.dynamic.quick_power_drop  != AMEC_INTF_GET_OVERSUBSCRIPTION() ||
-            G_opal_dynamic_table.dynamic.power_shift_ratio != G_sysConfigData.psr              ||
-            G_opal_dynamic_table.dynamic.power_cap_type    != G_master_pcap_data.source        ||
-            G_opal_dynamic_table.dynamic.min_power_cap     != G_master_pcap_data.soft_min_pcap ||
-            G_opal_dynamic_table.dynamic.max_power_cap     != G_master_pcap_data.max_pcap      ||
-            G_opal_dynamic_table.dynamic.current_power_cap != G_master_pcap_data.current_pcap  )
+    // check if previous change is not complete
+    else if( (G_opal_table_update_state == OPAL_TABLE_UPDATE_DYNAMIC_COPY) ||
+             (G_opal_dynamic_bce_req_scheduled && !(async_request_is_idle(&G_opal_dynamic_bce_req.request))) )
     {
-        if (G_opal_dynamic_table.dynamic.occ_state         != CURRENT_STATE())
-            TRAC_INFO("check_for_opal_updates: state changed from 0x%02X->0x%02X", G_opal_dynamic_table.dynamic.occ_state, CURRENT_STATE());
-        if (G_opal_dynamic_table.dynamic.quick_power_drop  != AMEC_INTF_GET_OVERSUBSCRIPTION())
-            TRAC_INFO("check_for_opal_updates: qpd changed from 0x%02X->0x%02X", G_opal_dynamic_table.dynamic.quick_power_drop, AMEC_INTF_GET_OVERSUBSCRIPTION());
-        if (G_opal_dynamic_table.dynamic.power_cap_type    != G_master_pcap_data.source        ||
-            G_opal_dynamic_table.dynamic.min_power_cap     != G_master_pcap_data.soft_min_pcap ||
-            G_opal_dynamic_table.dynamic.max_power_cap     != G_master_pcap_data.max_pcap      ||
-            G_opal_dynamic_table.dynamic.current_power_cap != G_master_pcap_data.current_pcap)
-            TRAC_INFO("check_for_opal_updates: power cap change");
+         if(L_num_bce_checks <= OPAL_DYNAMIC_UPDATE_BCE_RETRIES)
+         {
+             L_num_bce_checks++;
+         }
+         else
+         {
+             TRAC_ERR("check_for_opal_updates: BCE not idle %u times, done retrying", L_num_bce_checks);
+             l_log_crit_error = true;
+         }
+    }
 
-        update_dynamic_opal_data();
+    else if(G_opal_table_update_state == OPAL_TABLE_UPDATE_BCE_FAIL)
+    {
+         // BCE failed re-populate the data and retry the BCE if under retry count
+         if(L_num_bce_checks <= OPAL_DYNAMIC_UPDATE_BCE_RETRIES)
+         {
+             dynamic_data_change = true;
+         }
+         else
+         {
+             TRAC_ERR("check_for_opal_updates: BCE failed %u times, done retrying", L_num_bce_checks);
+             l_log_crit_error = true;
+         }
     }
 
-    // A throttle status change, notify host after copying dynamic OPAL data
-    if(throttle_change)
+    else  // check if any of the data changed
+    {
+        // check if processor throttle status changed if going to safe state check for reset status
+        // else just check for any change since not in safe state
+        if( isSafeStateRequested() )
+        {
+            if(G_opal_dynamic_table.dynamic.proc_throt_status != OCC_RESET)
+            {
+                dynamic_data_change = true;
+                TRAC_INFO("check_for_opal_updates: safe state processor throttle status change - 0x%02X->0x%02X",
+                           G_opal_dynamic_table.dynamic.proc_throt_status, G_amec_opal_proc_throt_reason);
+            }
+        }
+        else if(G_opal_dynamic_table.dynamic.proc_throt_status != G_amec_opal_proc_throt_reason)
+        {
+            dynamic_data_change = true;
+            TRAC_INFO("check_for_opal_updates: processor throttle status change - 0x%02X->0x%02X",
+                       G_opal_dynamic_table.dynamic.proc_throt_status, G_amec_opal_proc_throt_reason);
+        }
+
+        // check if memory throttle status or Quick Power Drop changed
+        if( (G_opal_dynamic_table.dynamic.mem_throt_status != G_amec_opal_mem_throt_reason) ||
+            (G_opal_dynamic_table.dynamic.quick_power_drop != AMEC_INTF_GET_OVERSUBSCRIPTION()) ) 
+        {
+            dynamic_data_change = true;
+            TRAC_INFO("check_for_opal_updates: memory throttle status - 0x%02X->0x%02X QPD - 0x%02X->0x%02X",
+                       G_opal_dynamic_table.dynamic.mem_throt_status, G_amec_opal_mem_throt_reason,
+                       G_opal_dynamic_table.dynamic.quick_power_drop, AMEC_INTF_GET_OVERSUBSCRIPTION());
+        }
+
+        // check for OCC state change
+        if(G_opal_dynamic_table.dynamic.occ_state != CURRENT_STATE())
+        {
+            dynamic_data_change = true;
+            TRAC_INFO("check_for_opal_updates: OCC state change 0x%02X->0x%02X",
+                       G_opal_dynamic_table.dynamic.occ_state, CURRENT_STATE());
+        }
+
+        // check for change in power cap data must look at slave copy
+        // do NOT use G_master_pcap_data as that is not populated on slaves
+        if( (G_opal_dynamic_table.dynamic.min_power_cap      != G_sysConfigData.pcap.hard_min_pcap) ||
+            (G_opal_dynamic_table.dynamic.max_power_cap      != G_sysConfigData.pcap.max_pcap)      ||
+            (G_opal_dynamic_table.dynamic.soft_min_power_cap != G_sysConfigData.pcap.soft_min_pcap) ||
+            (G_opal_dynamic_table.dynamic.power_shift_ratio  != G_sysConfigData.psr)                ||
+            (G_opal_dynamic_table.dynamic.power_cap_type     != G_sysConfigData.pcap.source)        ||
+            (G_opal_dynamic_table.dynamic.current_power_cap  != g_amec->pcap.active_node_pcap) )
+        {
+            dynamic_data_change = true;
+            TRAC_INFO("check_for_opal_updates: soft min Pcap = 0x%04X->0x%04X hard min Pcap = 0x%04X->0x%04X",
+                       G_opal_dynamic_table.dynamic.soft_min_power_cap, G_sysConfigData.pcap.soft_min_pcap,
+                       G_opal_dynamic_table.dynamic.min_power_cap, G_sysConfigData.pcap.hard_min_pcap);
+
+            TRAC_INFO("check_for_opal_updates: max Pcap = 0x%04X->0x%04X active Pcap = 0x%04X->0x%04X",
+                       G_opal_dynamic_table.dynamic.max_power_cap, G_sysConfigData.pcap.max_pcap,
+                       G_opal_dynamic_table.dynamic.current_power_cap, g_amec->pcap.active_node_pcap);
+
+            TRAC_INFO("check_for_opal_updates: Pcap PSR = %u->%u Pcap source = %u->%u",
+                       G_opal_dynamic_table.dynamic.power_shift_ratio, G_sysConfigData.psr,
+                       G_opal_dynamic_table.dynamic.power_cap_type, G_sysConfigData.pcap.source);
+        }
+    }  // else check for changes
+
+    // If there was a change copy to main memory and notify host when BCE finishes
+    if(dynamic_data_change)
+    {
+        G_opal_table_update_state = OPAL_TABLE_UPDATE_DYNAMIC_COPY;
+        update_dynamic_opal_data();
+
+        // if the BCE schedule fails the state will go back to IDLE, retry next time called
+        if(G_opal_table_update_state == OPAL_TABLE_UPDATE_IDLE)
+        {
+            G_opal_table_update_state = OPAL_TABLE_UPDATE_BCE_FAIL;
+            L_num_bce_checks++;
+        }
+        else
+        {
+            L_num_bce_checks = 0;
+        }
+    }
+    else if(l_log_crit_error)
     {
-        notify_host(INTR_REASON_OPAL_SHARED_MEM_CHANGE);
+         // stop trying to update dynamic data, this only really matters on OPAL systems so
+         // only log the error if OPAL
+         G_opal_table_update_state = OPAL_TABLE_UPDATE_CRITICAL_ERROR;
+
+         if(G_sysConfigData.system_type.kvm)
+         {
+             // Create and commit error
+             /* @
+              * @errortype
+              * @moduleid    PROC_CHECK_FOR_OPAL_UPDATES_MOD
+              * @reasoncode  OPAL_TABLE_UPDATE_ERROR
+              * @userdata1   0
+              * @userdata2   0
+              * @userdata4   ERC_GENERIC_TIMEOUT
+              * @devdesc     BCE request failure to update dynamic opal table
+              */
+             errlHndl_t l_errl = createErrl(PROC_CHECK_FOR_OPAL_UPDATES_MOD, // Module ID
+                                            OPAL_TABLE_UPDATE_ERROR,         // Reason code
+                                            ERC_GENERIC_TIMEOUT,             // Extended reason code
+                                            ERRL_SEV_UNRECOVERABLE,          // Severity
+                                            NULL,                            // Trace Buffers
+                                            DEFAULT_TRACE_SIZE,              // Trace Size
+                                            0,                               // Userdata1
+                                            0);                              // Userdata2
+
+            // Callout firmware
+            addCalloutToErrl(l_errl,
+                             ERRL_CALLOUT_TYPE_COMPONENT_ID,
+                             ERRL_COMPONENT_ID_FIRMWARE,
+                             ERRL_CALLOUT_PRIORITY_HIGH);
+
+            // Callout processor
+            addCalloutToErrl(l_errl,
+                             ERRL_CALLOUT_TYPE_HUID,
+                             G_sysConfigData.proc_huid,
+                             ERRL_CALLOUT_PRIORITY_MED);
+
+             commitErrl(&l_errl);
+         }
     }
 }
 
@@ -457,7 +623,7 @@ void check_for_opal_updates(void)
 void update_dynamic_opal_data (void)
 {
 
-    // Initialize the opal table in SRAM (sets valid bit)
+    // Initialize the dynamic opal table in SRAM
     populate_opal_dynamic_data();
 
     // copy sram image into mainstore HOMER
diff --git a/src/occ_405/proc/proc_pstate.h b/src/occ_405/proc/proc_pstate.h
index ac7de35..0126d1d 100755
--- a/src/occ_405/proc/proc_pstate.h
+++ b/src/occ_405/proc/proc_pstate.h
@@ -94,14 +94,14 @@ typedef struct __attribute__ ((packed))
     uint16_t             min_power_cap;
     uint16_t             max_power_cap;
     uint16_t             current_power_cap;
-
+    uint16_t             soft_min_power_cap;
 } opal_dynamic_t;
 
 // This size must be a multiple of 128
 typedef struct __attribute__ ((packed))
 {
-    opal_dynamic_t       dynamic;       // Dynamic OPAL parameters: 16B
-    uint8_t              pad[112];      // Reserved dynamic space: 112B
+    opal_dynamic_t       dynamic;       // Dynamic OPAL parameters: 18B
+    uint8_t              pad[110];      // Reserved dynamic space: 110B
 } opal_dynamic_table_t __attribute__ ((aligned (128)));
 
 #define PSTATE_ENTRIES 256    // number of generated PSTATES entries in OPAL table
@@ -125,6 +125,18 @@ extern uint8_t     G_desired_pstate[MAXIMUM_QUADS];
 extern opal_dynamic_table_t G_opal_dynamic_table;
 extern opal_static_table_t G_opal_static_table;
 
+// States for updating opal data in main memory
+typedef enum
+{
+    OPAL_TABLE_UPDATE_IDLE = 0x00,            // No dynmaic table update in process
+    OPAL_TABLE_UPDATE_DYNAMIC_COPY = 0x01,    // BCE scheduled to copy dynamic date to main memory
+    OPAL_TABLE_UPDATE_NOTIFY_HOST = 0x02,     // BCE copy finished notify host
+    OPAL_TABLE_UPDATE_BCE_FAIL = 0x03,        // BCE failed, retry
+    OPAL_TABLE_UPDATE_CRITICAL_ERROR = 0x04,  // Critical BCE error, stop trying to update dynamic data
+} OPAL_TABLE_UPDATE_STATE;
+
+extern volatile uint8_t G_opal_table_update_state;
+
 // Helper function to translate from Frequency to nearest Pstate
 Pstate proc_freq2pstate(uint32_t i_freq_mhz);
 
diff --git a/src/occ_405/pss/apss.c b/src/occ_405/pss/apss.c
index eb9c3d7..a4c00e0 100755
--- a/src/occ_405/pss/apss.c
+++ b/src/occ_405/pss/apss.c
@@ -141,6 +141,8 @@ void dumpHexString(const void *i_data, const unsigned int len, const char *strin
 // End Function Specification
 void do_apss_recovery(void)
 {
+    INCREMENT_ERR_HISTORY(ERR_INVALID_APSS_DATA);
+
     if (!G_apss_data_traced)
     {
         INTR_TRAC_ERR("detected invalid power data[%08x%08x]",
diff --git a/src/occ_405/state.c b/src/occ_405/state.c
index 5170c5b..a0efa22 100755
--- a/src/occ_405/state.c
+++ b/src/occ_405/state.c
@@ -43,9 +43,9 @@
 #include "amec_sys.h"
 
 // Maximum time to wait for a PGPE task before timeout
-#define WAIT_PGPE_TASK_TIMEOUT 200
-// Maximum time to wait for a PGPE task before timeout when transitioning to standby
-#define WAIT_PGPE_TASK_TIMEOUT_STDBY 50
+// must wait at least 1 tick time to ensure proc_data_control() runs to set
+// clips on active->obs state transition
+#define WAIT_PGPE_TASK_TIMEOUT (MICS_PER_TICK + 20)
 
 extern bool G_mem_monitoring_allowed;
 extern task_t G_task_table[TASK_END];  // Global task table
@@ -205,7 +205,8 @@ errlHndl_t SMGR_standby_to_characterization()
     Pstate      l_pstate;
     do
     {
-
+        // characterization state will have Pstate protocol enabled check that all data for active
+        // is available (frequency points)
         if( SMGR_MASK_ACTIVE_READY ==
             (SMGR_validate_get_valid_states() & SMGR_MASK_ACTIVE_READY))
         {
@@ -224,20 +225,12 @@ errlHndl_t SMGR_standby_to_characterization()
             else // successfully set clips; enable pstates, then start transition
             {
 
-                // update OPAL static table in main memory
-                if(G_sysConfigData.system_type.kvm)
-                {
-                    // upon succesful enablement of Pstate protocol on
-                    // PGPE update OPAL table with pstate information.
-                    proc_pstate_kvm_setup();
-                }
-
                 // Start pstates on PGPE and set Characterization as owner
                 rc = pgpe_start_suspend(PGPE_ACTION_PSTATE_START, PMCR_OWNER_CHAR);
 
                 if(rc)
                 {
-                    TRAC_ERR("SMGR: failed to start the pstate protocol on PGPE.");
+                    TRAC_ERR("SMGR: failed to start the pstate protocol for char owner on PGPE.");
                     break;
                 }
                 else // Clips set and pstates started successfully, start transition
@@ -307,7 +300,7 @@ errlHndl_t SMGR_all_to_standby()
     // if Psates in transition (a pgpe_start_suspend IPC call still running),
     // wait until it is settled up to WAIT_PGPE_TASK_TIMEOUT usec
     while( (G_proc_pstate_status == PSTATES_IN_TRANSITION) &&
-           (wait_time < WAIT_PGPE_TASK_TIMEOUT_STDBY) )
+           (wait_time < WAIT_PGPE_TASK_TIMEOUT) )
     {
         // wait until pgpe_start_suspend call is completed. Sleep enables context switching.
         ssx_sleep(SSX_MICROSECONDS(10));
@@ -317,7 +310,7 @@ errlHndl_t SMGR_all_to_standby()
     // check for timeout while waiting for pgpe_start_suspend() IPC completion
     if(wait_time >= WAIT_PGPE_TASK_TIMEOUT)
     {
-        TRAC_ERR("SMGR: Timeout waiting for Pstates start/suspend IPC task");
+        TRAC_ERR("SMGR_all_to_standby: Timeout waiting for Pstates start/suspend IPC task");
     }
     // Stop Pstates if enabled
     else if(G_proc_pstate_status == PSTATES_ENABLED)
@@ -325,7 +318,7 @@ errlHndl_t SMGR_all_to_standby()
         rc = pgpe_start_suspend(PGPE_ACTION_PSTATE_STOP, G_proc_pmcr_owner);
         if(rc)
         {
-            TRAC_ERR("SMGR: Failed to stop the pstate protocol on PGPE. rc[%08X]", rc);
+            TRAC_ERR("SMGR_all_to_standby: Failed to stop the pstate protocol on PGPE. rc[%08X]", rc);
         }
     }
 
@@ -365,7 +358,7 @@ errlHndl_t SMGR_characterization_to_observation()
         rc = pgpe_set_clip_blocking(l_pstate);
         if(rc)
         {
-            TRAC_ERR("SMGR: failed to tighten pstate clips.");
+            TRAC_ERR("SMGR_char_to_obs: failed to set pstate clip to legacy turbo rc[%08X]", rc);
             break;
         }
         else // clips set to legacy turbo; stop pstate protocol
@@ -373,7 +366,7 @@ errlHndl_t SMGR_characterization_to_observation()
             rc = pgpe_start_suspend(PGPE_ACTION_PSTATE_STOP, G_proc_pmcr_owner);
             if(rc)
             {
-                TRAC_ERR("SMGR: Failed to stop the pstate protocol on PGPE.");
+                TRAC_ERR("SMGR_char_to_obs: Failed to stop pstate protocol rc[%08X]", rc);
                 break;
             }
             else // Clips tightened successfully, and pstates disabled: perform transition
@@ -394,7 +387,7 @@ errlHndl_t SMGR_characterization_to_observation()
          * @errortype
          * @moduleid    MAIN_STATE_TRANSITION_MID
          * @reasoncode  INTERNAL_FAILURE
-         * @userdata1   none
+         * @userdata1   rc
          * @userdata4   ERC_STATE_FROM_CHR_TO_OBS_FAILURE
          * @devdesc     Failed changing from observation to characterization
          */
@@ -404,7 +397,7 @@ errlHndl_t SMGR_characterization_to_observation()
                                 ERRL_SEV_UNRECOVERABLE,             //Severity
                                 NULL,                               //Trace Buf
                                 DEFAULT_TRACE_SIZE,                 //Trace Size
-                                0,                                  //userdata1
+                                rc,                                 //userdata1
                                 0);                                 //userdata2
 
         // Callout firmware
@@ -451,25 +444,17 @@ errlHndl_t SMGR_observation_to_characterization()
 
         if(rc)
         {
-            TRAC_ERR("SMGR: failed to set pstate clips.");
+            TRAC_ERR("SMGR_obs_to_char: failed to set pstate clips rc[%08X]", rc);
             break;
         }
         else // successfully set clips; enable pstates, then start transition
         {
-            // update OPAL static table in main memory
-            if(G_sysConfigData.system_type.kvm)
-            {
-                // upon succesful enablement of Pstate protocol on
-                // PGPE update OPAL table with pstate information.
-                proc_pstate_kvm_setup();
-            }
-
             // Start pstates on PGPE and set Characterization as owner
             rc = pgpe_start_suspend(PGPE_ACTION_PSTATE_START, PMCR_OWNER_CHAR);
 
             if(rc)
             {
-                TRAC_ERR("SMGR: failed to start the pstate protocol on PGPE.");
+                TRAC_ERR("SMGR_obs_to_char: failed to start pstate protocol rc[%08X]", rc);
                 break;
             }
             else // Clips set successfully and pstates enabled; complete transition
@@ -490,7 +475,7 @@ errlHndl_t SMGR_observation_to_characterization()
          * @errortype
          * @moduleid    MAIN_STATE_TRANSITION_MID
          * @reasoncode  INTERNAL_FAILURE
-         * @userdata1   none
+         * @userdata1   rc
          * @userdata4   ERC_STATE_FROM_OBS_TO_CHR_FAILURE
          * @devdesc     Failed changing from observation to characterization
          */
@@ -500,7 +485,7 @@ errlHndl_t SMGR_observation_to_characterization()
                                 ERRL_SEV_UNRECOVERABLE,             //Severity
                                 NULL,                               //Trace Buf
                                 DEFAULT_TRACE_SIZE,                 //Trace Size
-                                0,                                  //userdata1
+                                rc,                                 //userdata1
                                 0);                                 //userdata2
 
         // Callout firmware
@@ -552,8 +537,7 @@ errlHndl_t SMGR_observation_to_active()
 
             if(l_rc)
             {
-                TRAC_ERR("SMGR: Failed to switch to Active state because of a "
-                         "failure to set clip pstates");
+                TRAC_ERR("SMGR_obs_to_active: Set Pstate clips failed rc[%08X]", l_rc);
                 break;
             }
 
@@ -572,8 +556,7 @@ errlHndl_t SMGR_observation_to_active()
                 }
                 if(l_rc)
                 {
-                    TRAC_ERR("SMGR: Failed to switch to Active state because of a "
-                             "failure to start the pstate protocol on PGPE.");
+                    TRAC_ERR("SMGR_obs_to_active: Failed to start pstate protocol rc[%08X]", l_rc);
                     break;
                 }
             }
@@ -588,7 +571,7 @@ errlHndl_t SMGR_observation_to_active()
                     l_rc = 1;
                     if(FALSE == L_error_logged)
                     {
-                        TRAC_ERR("SMGR: Timeout waiting for Pstates to be enabled, "
+                        TRAC_ERR("SMGR_obs_to_active: Timeout waiting for Pstates to be enabled, "
                                  "chips_present[%02x], Cores Present [%08x]",
                                  G_sysConfigData.is_occ_present,
                                  (uint32_t) ((in64(OCB_CCSR)) >> 32));
@@ -603,6 +586,8 @@ errlHndl_t SMGR_observation_to_active()
             // to transition to active state.
             if(proc_is_hwpstate_enabled() )
             {
+                TRAC_IMP("SMGR_obs_to_active: Pstates are enabled, continuing with state trans");
+
                 L_error_logged = FALSE;
 
                 // Set the RTL Flags to indicate which tasks can run
@@ -611,15 +596,8 @@ errlHndl_t SMGR_observation_to_active()
                 rtl_clr_run_mask_deferred(RTL_FLAG_OBS);
                 rtl_set_run_mask_deferred(RTL_FLAG_ACTIVE);
 
-                // Pstates enabled, update OPAL static table in main memory
-                if(G_sysConfigData.system_type.kvm)
-                {
-                    TRAC_IMP("SMGR: Pstates are enabled, continuing with state trans");
-
-                    // upon succesful enablement of Pstate protocol on
-                    // PGPE update OPAL table with pstate information.
-                    proc_pstate_kvm_setup();
-                }
+                // Pstates enabled, update OPAL static table in main memory with pState info
+                proc_pstate_kvm_setup();
 
                 // Set the actual STATE now that we have finished everything else
                 CURRENT_STATE() = OCC_STATE_ACTIVE;
@@ -684,6 +662,8 @@ errlHndl_t SMGR_characterization_to_active()
     errlHndl_t  l_errlHndl = NULL;
     static bool L_error_logged = FALSE;  // To prevent trace and error log happened over and over
 
+    TRAC_IMP("SMGR: Characterization to Active Transition Started");
+
     do
     {
         // change PMCR ownership via an IPC call to PGPE based on system type.
@@ -697,27 +677,27 @@ errlHndl_t SMGR_characterization_to_active()
         }
         if(rc)
         {
-            TRAC_ERR("SMGR: Failed to change PMCR ownership on PGPE.");
+            TRAC_ERR("SMGR_char_to_active: Failed to change PMCR ownership rc[%08X]", rc);
             break;
         }
 
-        // Have all data we need to go to active state
-        if(((DATA_get_present_cnfgdata() & SMGR_VALIDATE_DATA_ACTIVE_MASK) ==
-            SMGR_VALIDATE_DATA_ACTIVE_MASK)   &&
-           // and Psate protocol is enabled
-           proc_is_hwpstate_enabled() && G_sysConfigData.system_type.kvm)
+        // Wait for ownership change to complete
+        SsxTimebase start = ssx_timebase_get();
+        SsxInterval timeout =  SSX_SECONDS(5);
+        while( ! proc_is_hwpstate_enabled() )
         {
-            TRAC_IMP("SMGR: Pstates are enabled, continuing with state trans");
+            if ((ssx_timebase_get() - start) > timeout)
+            {
+                rc = 1;
+                TRAC_ERR("SMGR_char_to_active: Timeout waiting for PMCR ownership change");
+                break;
+            }
+            ssx_sleep(SSX_MICROSECONDS(10));
         }
 
-        // Check if all conditions are met to transition to active state.  If
-        // they aren't, then log an error and fail to change state.
-        if( (SMGR_MASK_ACTIVE_READY ==
-             (SMGR_validate_get_valid_states() & SMGR_MASK_ACTIVE_READY))
-            && proc_is_hwpstate_enabled() )
+        if(proc_is_hwpstate_enabled())
         {
             L_error_logged = FALSE;
-            TRAC_IMP("SMGR: Characterization to Active Transition Started");
 
             // Set the RTL Flags to indicate which tasks can run
             //   - Clear OBSERVATION b/c not in CHARACTERIZATION State
@@ -738,7 +718,7 @@ errlHndl_t SMGR_characterization_to_active()
          * @errortype
          * @moduleid    MAIN_STATE_TRANSITION_MID
          * @reasoncode  INTERNAL_FAILURE
-         * @userdata1   SMGR_MASK_ACTIVE_READY
+         * @userdata1   rc
          * @userdata2   valid states
          * @userdata4   ERC_STATE_FROM_CHR_TO_ACT_FAILURE
          * @devdesc     Failed changing from characterization to active
@@ -749,7 +729,7 @@ errlHndl_t SMGR_characterization_to_active()
                                 ERRL_SEV_UNRECOVERABLE,              //Severity
                                 NULL,                                //Trace Buf
                                 DEFAULT_TRACE_SIZE,                  //Trace Size
-                                SMGR_MASK_ACTIVE_READY,              //userdata1
+                                rc,                                  //userdata1
                                 SMGR_validate_get_valid_states());   //userdata2
 
         // Callout firmware
@@ -797,7 +777,7 @@ errlHndl_t SMGR_active_to_observation()
         // check for timeout while waiting for pgpe_start_suspend() IPC completion
         if(wait_time > WAIT_PGPE_TASK_TIMEOUT)
         {
-            TRAC_ERR("SMGR: Timeout waiting for G_active_to_observation_ready flag.");
+            TRAC_ERR("SMGR_act_to_obs: Timeout waiting for G_active_to_observation_ready flag.");
 
             /* @
              * @errortype
@@ -842,10 +822,10 @@ errlHndl_t SMGR_active_to_observation()
             wait_time += 10;
         }
 
-        // check for timeout while waiting for pgpe_start_suspend() IPC completion
+        // check for timeout while waiting for Pstate clips IPC completion
         if(wait_time > WAIT_PGPE_TASK_TIMEOUT)
         {
-            TRAC_ERR("SMGR: Timeout waiting for clip update IPC task");
+            TRAC_ERR("SMGR_act_to_obs: Timeout waiting for clip update IPC task");
 
             /* @
              * @errortype
@@ -877,7 +857,7 @@ errlHndl_t SMGR_active_to_observation()
         rc = pgpe_start_suspend(PGPE_ACTION_PSTATE_STOP, G_proc_pmcr_owner);
         if(rc)
         {
-            TRAC_ERR("SMGR: failed to stop the pstate protocol on PGPE.");
+            TRAC_ERR("SMGR_act_to_obs: failed to stop the pstate protocol on PGPE.");
             break;
         }
         else // Pstates Disabled and clips set successfully, perform state transition
@@ -900,18 +880,10 @@ errlHndl_t SMGR_active_to_observation()
 
     if(rc)
     {
-        TRAC_ERR("SMGR: Failed to switch to Observation state");
+        TRAC_ERR("SMGR: Failed with rc = %d to switch to Observation state", rc);
     }
     else
     {
-        // Pstates disabled, update OPAL static table in main memory
-        if(G_sysConfigData.system_type.kvm)
-        {
-            // upon succesful enablement of Pstate protocol on
-            // PGPE update OPAL table with pstate information.
-            proc_pstate_kvm_setup();
-        }
-
         TRAC_IMP("SMGR: Active to Observation Transition Completed");
     }
 
@@ -942,7 +914,7 @@ errlHndl_t SMGR_active_to_characterization()
 
         if(rc)
         {
-            TRAC_ERR("SMGR: failed to set pstate clips.");
+            TRAC_ERR("SMGR_act_to_char: failed to set pstate clips.");
             break;
         }
         else // clips set successfully, keep pstates enabled, but change ownership
@@ -954,16 +926,32 @@ errlHndl_t SMGR_active_to_characterization()
                 TRAC_ERR("SMGR: failed to change PMCR ownership.");
                 break;
             }
-            else // Pstates Disabled and clips set successfully, perform state transition
+            else // Request successfully scheduled on PGPE now verify it completed
             {
-                // Set the RTL Flags to indicate which tasks can run
-                //   - Set OBSERVATION RTL flags for Characterization State
-                //   - Clear ACTIVE b/c not in ACTIVE State
-                rtl_clr_run_mask_deferred(RTL_FLAG_ACTIVE);
-                rtl_set_run_mask_deferred(RTL_FLAG_OBS);
-
-                // Set the actual STATE now that we have finished everything else
-                CURRENT_STATE() = OCC_STATE_CHARACTERIZATION;
+               // Wait for ownership change to complete
+               SsxTimebase start = ssx_timebase_get();
+               SsxInterval timeout =  SSX_SECONDS(5);
+               while( ! proc_is_hwpstate_enabled() )
+               {
+                  if ((ssx_timebase_get() - start) > timeout)
+                  {
+                      rc = 1;
+                      TRAC_ERR("SMGR_active_to_char: Timeout waiting for PMCR ownership change");
+                      break;
+                  }
+                  ssx_sleep(SSX_MICROSECONDS(10));
+               }
+               if(proc_is_hwpstate_enabled())
+               {
+                   // Set the RTL Flags to indicate which tasks can run
+                   //   - Set OBSERVATION RTL flags for Characterization State
+                   //   - Clear ACTIVE b/c not in ACTIVE State
+                   rtl_clr_run_mask_deferred(RTL_FLAG_ACTIVE);
+                   rtl_set_run_mask_deferred(RTL_FLAG_OBS);
+
+                   // Set the actual STATE now that we have finished everything else
+                   CURRENT_STATE() = OCC_STATE_CHARACTERIZATION;
+               }
             }
         }
     } while (0);
@@ -996,14 +984,6 @@ errlHndl_t SMGR_active_to_characterization()
     }
     else
     {
-        // Pstates disabled, update OPAL static table in main memory
-        if(G_sysConfigData.system_type.kvm)
-        {
-            // upon succesful enablement of Pstate protocol on
-            // PGPE update OPAL table with pstate information.
-            proc_pstate_kvm_setup();
-        }
-
         TRAC_IMP("SMGR: Active to Characterization Transition Completed");
     }