Adding Thread Query feature to p9_thread_control HWP (Level 2)

Change-Id: I4dc29ba688fa314a39cd2aa7faa3e2d90984b8bb
RTC:149669
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/22410
Tested-by: Jenkins Server
Tested-by: Hostboot CI
Tested-by: PPE CI
Reviewed-by: Joseph J. McGill <jmcgill@us.ibm.com>
Reviewed-by: RAJA DAS <rajadas2@in.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/22548
Tested-by: FSP CI Jenkins
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

5 files changed, 271 insertions, 117 deletions

diff --git a/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.C b/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.C
index d1cc2c1cb..2be80dd12 100644
--- a/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.C
+++ b/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.C
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -22,7 +22,7 @@
 ///    Starts instructions on 1 core, thread 0.
 ///    Thread 0 will be started at CIA scan flush value of 0.
 //
-// *HWP HWP Owner: Michael Dye <dyem@us.ibm.com>
+// *HWP HWP Owner: Nick Klazynski <jklazyns@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
 // *HWP Level: 2
@@ -40,19 +40,20 @@ extern "C"
 ///
 /// p9_sbe_instruct_start HWP entry point (Defined in .H file)
 ///
-    fapi2::ReturnCode p9_sbe_instruct_start(const fapi2::Target<fapi2::TARGET_TYPE_CORE>
-                                            & i_target)
+    fapi2::ReturnCode p9_sbe_instruct_start(
+        const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target)
     {
-        //Mark Entry
-        FAPI_INF("Entering ...");
+        fapi2::buffer<uint64_t> l_rasStatusReg(0);
+        uint64_t l_state = 0;
+        FAPI_DBG("Entering ...");
 
-        FAPI_INF("Starting thread 0 with bitset 1000 at address 0x0...");
-        FAPI_TRY(p9_thread_control(i_target, 8, PTC_CMD_START, false),
-                 "Failed when calling p9_thread_control thread 0 start");
-        //Mark Exit
-        FAPI_INF("Exiting ...");
+        FAPI_INF("Starting instruction on thread 0");
+        FAPI_TRY(p9_thread_control(i_target, 0b1000, PTC_CMD_START, false,
+                                   l_rasStatusReg, l_state),
+                 "p9_sbe_instruct_start: p9_thread_control() returns an error");
 
     fapi_try_exit:
+        FAPI_DBG("Exiting ...");
         return fapi2::current_err;
     }
 
diff --git a/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.H b/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.H
index c79384e77..cc4303b35 100644
--- a/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.H
+++ b/src/import/chips/p9/procedures/hwp/core/p9_sbe_instruct_start.H
@@ -7,7 +7,7 @@
 /*                                                                        */
 /* EKB Project                                                            */
 /*                                                                        */
-/* COPYRIGHT 2015                                                         */
+/* COPYRIGHT 2015,2016                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -20,7 +20,7 @@
 /// @file p9_sbe_instruct_start.C
 /// @brief  Placeholder for overrides needed to step the core from cache-contained execution to expand to memory
 ///
-// *HWP HWP Owner: Michael Dye <dyem@us.ibm.com>
+// *HWP HWP Owner: Nick Klazynski <jklazyns@us.ibm.com>
 // *HWP FW Owner: Thi Tran <thi@us.ibm.com>
 // *HWP Team: Nest
 // *HWP Level: 2
@@ -38,7 +38,7 @@
 
 // function pointer typedef definition for HWP call support
 typedef fapi2::ReturnCode
-(*p9_sbe_instruct_start_FP_t)(const fapi2::Target<fapi2::TARGET_TYPE_CORE>& );
+(*p9_sbe_instruct_start_FP_t)(const fapi2::Target<fapi2::TARGET_TYPE_CORE>&);
 
 //------------------------------------------------------------------------------
 // Function prototypes
@@ -49,12 +49,15 @@ extern "C"
 // Hardware Procedure
 //------------------------------------------------------------------------------
 ///
-/// @brief Calls thread_control to start thread 0
-/// @param[in] i_target Reference to core target
-/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, RC_P9_THREAD_CONTROL_START_FAIL if start command failed,
-///         else error code
+/// @brief Calls thread_control to start instruction on thread 0.
+///        This is to be called during IPL (istep 5.2)
 ///
-    fapi2::ReturnCode p9_sbe_instruct_start(const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target);
+/// @param[in] i_target   Reference to core target
+///
+/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code
+///
+    fapi2::ReturnCode p9_sbe_instruct_start(
+        const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target);
 
 } // extern "C"
 
diff --git a/src/import/chips/p9/procedures/hwp/core/p9_thread_control.C b/src/import/chips/p9/procedures/hwp/core/p9_thread_control.C
index 2a792e583..9f791cff5 100644
--- a/src/import/chips/p9/procedures/hwp/core/p9_thread_control.C
+++ b/src/import/chips/p9/procedures/hwp/core/p9_thread_control.C
@@ -22,7 +22,7 @@
 ///
 
 // *HWP HWP Owner: Nick Klazynski <jklazyns@us.ibm.com>
-// *HWP FW Owner:  Brian Silver <bsilver@us.ibm.com>
+// *HWP FW Owner:  Thi Tran <thi@us.ibm.com>
 // *HWP Team:  Quad
 // *HWP Level: 2
 // Current Status: Only start function tested as working
@@ -67,44 +67,49 @@ static const uint64_t g_control_reg_map[] =
 
 fapi2::ReturnCode p9_thread_control_sreset(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck);
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg);
 
 fapi2::ReturnCode p9_thread_control_start(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck);
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg);
 
 fapi2::ReturnCode p9_thread_control_stop(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck);
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg);
 
 fapi2::ReturnCode p9_thread_control_step(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck);
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg);
+
+fapi2::ReturnCode p9_thread_control_query(
+    const fapi2::Target<TARGET_TYPE_CORE>& i_target,
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    uint64_t& o_state);
 
 //--------------------------------------------------------------------------
 /// @brief threads_running : static funtion to encapsulate the running state
 /// @param[in] i_target core target
 /// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
-/// @param[out] o_ok true iff the threads are running
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+/// @param[out] o_ok true if the threads are running
 /// @return FAPI2_RC_SUCCESS if the underlying hw operations succeeded
 //--------------------------------------------------------------------------
 static inline fapi2::ReturnCode threads_running(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, bool& o_ok)
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    bool& o_ok)
 {
     // Running is defined as not in maint mode and not quiesced.
-    const uint64_t l_running_mask =
-        (g_control_reg_map[i_threads] >> CORE_MAINT_MODE) |
-        (g_control_reg_map[i_threads] >> THREAD_QUIESCED);
-    FAPI_DBG("running_mask: 0x%lx", l_running_mask);
-
-    // Get the C_RAS_STATUS.
-    fapi2::buffer<uint64_t> l_ras_status;
-    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, l_ras_status),
-             "threads_running: ERROR checking C_RAS_STATUS bits for threads 0x%x", i_threads);
-
-    // If there are any threads with either the maint or quiesced bit set, we're not running.
-    o_ok = ((l_ras_status & l_running_mask) == 0);
+    uint64_t l_state = 0;
+    FAPI_TRY(p9_thread_control_query(i_target, i_threads, o_rasStatusReg, l_state),
+             "threads_running(): p9_thread_control_query() returns an error.");
+    o_ok = (l_state & THREAD_STATE_RUNNING);
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -114,21 +119,20 @@ fapi_try_exit:
 /// @brief threads_in_maint : static funtion to encapsulate the maint state
 /// @param[in] i_target core target
 /// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
-/// @param[out] o_ok true iff the threads are in maint mode
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+/// @param[out] o_ok true if the threads are in maint mode
 /// @return FAPI2_RC_SUCCESS if the underlying hw operations succeeded
 //--------------------------------------------------------------------------
 static inline fapi2::ReturnCode threads_in_maint(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, bool& o_ok)
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    bool& o_ok)
 {
-    const uint64_t l_maint_mask = (g_control_reg_map[i_threads] >> CORE_MAINT_MODE);
-    FAPI_DBG("maint_mask: 0x%lx", l_maint_mask);
-
-    fapi2::buffer<uint64_t> l_ras_status;
-    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, l_ras_status),
-             "threads_in_maint: ERROR checking C_RAS_STATUS bits for threads 0x%x", i_threads);
-
-    o_ok = ((l_ras_status & l_maint_mask) == l_maint_mask);
+    uint64_t l_state = 0;
+    FAPI_TRY(p9_thread_control_query(i_target, i_threads, o_rasStatusReg, l_state),
+             "threads_in_maint(): p9_thread_control_query() returns an error.");
+    o_ok = (l_state & THREAD_STATE_MAINT);
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -138,24 +142,21 @@ fapi_try_exit:
 /// @brief all_threads_stopped : static funtion to encapsulate the stopped state
 /// @param[in] i_target core target
 /// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
-/// @param[out] o_ok true iff the threads are stopped
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+/// @param[out] o_ok true if the threads are stopped
 /// @return FAPI2_RC_SUCCESS if the underlying hw operations succeeded
 //--------------------------------------------------------------------------
 static inline fapi2::ReturnCode threads_stopped(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, bool& o_ok)
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    bool& o_ok)
 {
     // Running is defined as not in maint mode and not quiesced.
-    const uint64_t l_running_mask = (g_control_reg_map[i_threads] >> CORE_MAINT_MODE) |
-                                    (g_control_reg_map[i_threads] >> THREAD_QUIESCED);
-    FAPI_DBG("running_mask: 0x%lx", l_running_mask);
-
-    fapi2::buffer<uint64_t> l_ras_status;
-    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, l_ras_status),
-             "threads_stopped: ERROR checking C_RAS_STATUS bits for threads 0x%x", i_threads);
-
-    // If there are any threads with either the maint or quiesced bit clear, we're running.
-    o_ok = ((l_ras_status & l_running_mask) == l_running_mask);
+    uint64_t l_state = 0;
+    FAPI_TRY(p9_thread_control_query(i_target, i_threads, o_rasStatusReg, l_state),
+             "threads_stopped(): p9_thread_control_query() returns an error.");
+    o_ok = (l_state & THREAD_STATE_STOP);
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -166,22 +167,20 @@ fapi_try_exit:
 /// complete state
 /// @param[in] i_target core target
 /// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
-/// @param[out] o_ok true iff the threads are done stepping
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+/// @param[out] o_ok true if the threads are done stepping
 /// @return FAPI2_RC_SUCCESS if the underlying hw operations succeeded
 //--------------------------------------------------------------------------
 static inline fapi2::ReturnCode threads_step_done(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, bool& o_ok)
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    bool& o_ok)
 {
-    const uint64_t step_done = (g_control_reg_map[i_threads] >> STEP_SUCCESS);
-    FAPI_DBG("step_done (mask): 0x%lx", step_done);
-
-    fapi2::buffer<uint64_t> l_ras_status;
-    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, l_ras_status),
-             "threads_step_done: ERROR checking C_RAS_STATUS bits for threads 0x%x",
-             i_threads);
-
-    o_ok = ((l_ras_status & step_done) == step_done);
+    uint64_t l_state = 0;
+    FAPI_TRY(p9_thread_control_query(i_target, i_threads, o_rasStatusReg, l_state),
+             "threads_step_done(): p9_thread_control_query() returns an error.");
+    o_ok = (l_state & THREAD_STATE_ISTEP_SUCCESS);
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -192,26 +191,20 @@ fapi_try_exit:
 /// ready state
 /// @param[in] i_target core target
 /// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
-/// @param[out] o_ok true iff the threads are ready to step
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+/// @param[out] o_ok true if the threads are ready to step
 /// @return FAPI2_RC_SUCCESS if the underlying hw operations succeeded
 //--------------------------------------------------------------------------
 static inline fapi2::ReturnCode threads_step_ready(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, bool& o_ok)
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    bool& o_ok)
 {
-    // Check for maint, quiesced and ICT empty.
-    const uint64_t l_step_ready = (g_control_reg_map[i_threads] >> CORE_MAINT_MODE) |
-                                  (g_control_reg_map[i_threads] >> THREAD_QUIESCED) |
-                                  (g_control_reg_map[i_threads] >> ICT_EMPTY);
-
-    FAPI_DBG("step_ready (mask): 0x%lx", l_step_ready);
-
-    fapi2::buffer<uint64_t> l_ras_status;
-    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, l_ras_status),
-             "threads_step_ready: ERROR checking C_RAS_STATUS bits for threads 0x%x",
-             i_threads);
-
-    o_ok = ((l_ras_status & l_step_ready) == l_step_ready);
+    uint64_t l_state = 0;
+    FAPI_TRY(p9_thread_control_query(i_target, i_threads, o_rasStatusReg, l_state),
+             "threads_step_ready(): p9_thread_control_query() returns an error.");
+    o_ok = (l_state & THREAD_STATE_ISTEP_READY);
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -223,26 +216,40 @@ fapi_try_exit:
 fapi2::ReturnCode p9_thread_control(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
     const uint8_t i_threads, const ThreadCommands i_command,
-    const bool i_warncheck)
+    const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    uint64_t& o_state)
 {
-    FAPI_INF("p9_thread_control : Start (core) threads: 0x%x)", i_threads);
+    FAPI_INF("p9_thread_control : Core threads: 0x%x, Command %u", i_threads, i_command);
+
+    // Output state is only valid for PTC_CMD_QUERY command
+    o_state = 0;
 
     switch(i_command)
     {
         case PTC_CMD_SRESET:
-            FAPI_TRY(p9_thread_control_sreset(i_target, i_threads, i_warncheck));
+            FAPI_TRY(p9_thread_control_sreset(i_target, i_threads, i_warncheck,
+                                              o_rasStatusReg));
             break;
 
         case PTC_CMD_START:
-            FAPI_TRY(p9_thread_control_start(i_target, i_threads, i_warncheck));
+            FAPI_TRY(p9_thread_control_start(i_target, i_threads, i_warncheck,
+                                             o_rasStatusReg));
             break;
 
         case PTC_CMD_STOP:
-            FAPI_TRY(p9_thread_control_stop(i_target, i_threads, i_warncheck));
+            FAPI_TRY(p9_thread_control_stop(i_target, i_threads, i_warncheck,
+                                            o_rasStatusReg));
             break;
 
         case PTC_CMD_STEP:
-            FAPI_TRY(p9_thread_control_step(i_target, i_threads, i_warncheck));
+            FAPI_TRY(p9_thread_control_step(i_target, i_threads, i_warncheck,
+                                            o_rasStatusReg));
+            break;
+
+        case PTC_CMD_QUERY:
+            FAPI_TRY(p9_thread_control_query(i_target, i_threads,
+                                             o_rasStatusReg, o_state));
             break;
     };
 
@@ -254,14 +261,122 @@ fapi_try_exit:
 }
 
 //--------------------------------------------------------------------------
+/// @brief Utility subroutine to query the state of a thread(s).
+///
+/// @param[in] i_target      Reference to core target
+/// @param[in] i_threads     Desired thread bits set
+///                             0b0000         No thread (No-op)
+///                             0b1000         Thread 0
+///                             0b0100         Thread 1
+///                             0b0010         Thread 2
+///                             0b0001         Thread 3
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
+///                             Only valid for PTC_CMD_QUERY command.
+/// @param[out] o_state      Current thread state. See THREAD_STATE bit
+///                          definitions in header file.
+///
+/// @return FAPI2_RC_SUCCESS if operation was successful, else error.
+//--------------------------------------------------------------------------
+fapi2::ReturnCode p9_thread_control_query(
+    const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target,
+    const uint8_t i_threads,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
+    uint64_t& o_state)
+{
+    FAPI_DBG("Entering: Thread bit set %u", i_threads);
+
+    // Initializing
+    o_state = 0;
+
+    // Setup mask values
+    const uint64_t l_running_mask =
+        (g_control_reg_map[i_threads] >> CORE_MAINT_MODE) |
+        (g_control_reg_map[i_threads] >> THREAD_QUIESCED);
+    const uint64_t l_step_ready_mask =
+        (g_control_reg_map[i_threads] >> CORE_MAINT_MODE) |
+        (g_control_reg_map[i_threads] >> THREAD_QUIESCED) |
+        (g_control_reg_map[i_threads] >> ICT_EMPTY);
+
+    // Get C_RAS_STATUS reg
+    FAPI_TRY(fapi2::getScom(i_target, C_RAS_STATUS, o_rasStatusReg),
+             "p9_thread_control_query(): getScom() returns an error, "
+             "Addr C_RAS_STATUS 0x%.16llX", C_RAS_STATUS);
+
+    // Note: all threads must meet a given condition in order for the
+    //       bit to be set.
+    // Set THREAD_STATE_RUNNING
+    // Running is defined as not in maint mode and not quiesced.
+    if ( ((o_rasStatusReg & l_running_mask) == 0) )
+    {
+        o_state |= THREAD_STATE_RUNNING;
+    }
+    // Stop is defined as in maint mode and in quiesced.
+    else if ( ((o_rasStatusReg & l_running_mask) == l_running_mask) )
+    {
+        o_state |= THREAD_STATE_STOP;
+    }
+
+    // Check for THREAD_STATE_MAINT
+    if ( o_rasStatusReg &
+         g_control_reg_map[i_threads] >> CORE_MAINT_MODE )
+    {
+        o_state |= THREAD_STATE_MAINT;
+    }
+
+    // Check for THREAD_STATE_QUIESCED
+    if ( o_rasStatusReg &
+         g_control_reg_map[i_threads] >> THREAD_QUIESCED )
+    {
+        o_state |= THREAD_STATE_QUIESCED;
+    }
+
+    // Check for THREAD_STATE_ICT_EMPTY
+    if ( o_rasStatusReg &
+         g_control_reg_map[i_threads] >> ICT_EMPTY )
+    {
+        o_state |= THREAD_STATE_ICT_EMPTY;
+    }
+
+    // Check for THREAD_STATE_LSU_QUIESCED
+    if ( o_rasStatusReg &
+         g_control_reg_map[i_threads] >> LSU_QUIESCED )
+    {
+        o_state |= THREAD_STATE_LSU_QUIESCED;
+    }
+
+    // Check for THREAD_STATE_ISTEP_SUCCESS
+    if ( o_rasStatusReg &
+         g_control_reg_map[i_threads] >> STEP_SUCCESS )
+    {
+        o_state |= THREAD_STATE_ISTEP_SUCCESS;
+    }
+
+    // Check for THREAD_STATE_ISTEP_READY
+    // All maint, quiesced and ICT empty must be set.
+    if ( ((o_rasStatusReg & l_step_ready_mask) == l_step_ready_mask) )
+    {
+        o_state |= THREAD_STATE_ISTEP_READY;
+    }
+
+    FAPI_DBG("C_RAS_STATUS: 0x%.16llX, Thread state 0x%.16llX",
+             o_rasStatusReg, o_state);
+
+fapi_try_exit:
+    FAPI_DBG("Exiting");
+    return fapi2::current_err;
+}
+
+//--------------------------------------------------------------------------
 /// @brief p9_thread_control: utility subroutine to control thread state
 //-------------------------------------------------------------------------
 fapi2::ReturnCode p9_thread_control(
     const fapi2::Target<TARGET_TYPE_EX>& i_target,
     const uint8_t i_threads,
     const ThreadCommands i_command,
+    fapi2::buffer<uint64_t>& o_rasStatusReg,
     const bool i_warncheck)
 {
+    uint64_t l_state = 0;
     FAPI_INF("p9_thread_control : Start (ex) threads: 0x%x)", i_threads);
 
     // Grab the normal core children and iterate over them.
@@ -278,7 +393,8 @@ fapi2::ReturnCode p9_thread_control(
 
         if (l_thread_set != 0)
         {
-            FAPI_TRY(p9_thread_control(*coreItr, l_thread_set, i_command, i_warncheck));
+            FAPI_TRY(p9_thread_control(*coreItr, l_thread_set, i_command,
+                                       i_warncheck, o_rasStatusReg, l_state));
         }
     }
 
@@ -292,13 +408,15 @@ fapi_try_exit:
 /// @param[in] i_target core target
 /// @param[in] i_threads normal core thread bitset (0b0000..0b1111)
 /// @param[in] i_warncheck convert pre/post checks errors to warnings
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
 /// @return FAPI2_RC_SUCCESS if operation was successful,
 ///         RC_P9_THREAD_CONTROL_SRESET_FAIL if the threads aren't running,
 ///         else error
 //--------------------------------------------------------------------------
 fapi2::ReturnCode p9_thread_control_sreset(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck)
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg)
 {
     FAPI_DBG("p9_thread_control_sreset : Initiating sreset command to core PC logic for threads 0x%x",
              i_threads);
@@ -317,7 +435,7 @@ fapi2::ReturnCode p9_thread_control_sreset(
     // TODO: Check for instructions having been executed?
     {
         bool l_running = false;
-        FAPI_TRY(threads_running(i_target, i_threads, l_running),
+        FAPI_TRY(threads_running(i_target, i_threads, o_rasStatusReg, l_running),
                  "p9_thread_control_sreset: unable to determine if threads are running. threads: 0x%x",
                  i_threads);
 
@@ -342,13 +460,15 @@ fapi_try_exit:
 /// @param[in] i_target core target
 /// @param[in] i_threads normal core thread bitset (0b0000..0b1111)
 /// @param[in] i_warncheck convert pre/post checks errors to warnings
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
 /// @return FAPI2_RC_SUCCESS if operation was successful,
 ///          RC_P9_THREAD_CONTROL_START_FAIL if start command failed,
 ///         else error
 //--------------------------------------------------------------------------
 fapi2::ReturnCode p9_thread_control_start(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck)
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg)
 {
     FAPI_DBG("p9_thread_control_start : Initiating start command to core PC logic for threads 0x%x",
              i_threads);
@@ -356,7 +476,7 @@ fapi2::ReturnCode p9_thread_control_start(
     // Preconditions: Only valid when in maint mode
     {
         bool l_in_maint = false;
-        FAPI_TRY(threads_in_maint(i_target, i_threads, l_in_maint),
+        FAPI_TRY(threads_in_maint(i_target, i_threads, o_rasStatusReg, l_in_maint),
                  "p9_thread_control_start: unable to determine if threads are in maint mode. threads: 0x%x",
                  i_threads);
 
@@ -383,7 +503,7 @@ fapi2::ReturnCode p9_thread_control_start(
     //       Verify understanding and desire for this funtionality before implementing in all thread_control functions
     {
         bool l_running = false;
-        FAPI_TRY(threads_running(i_target, i_threads, l_running),
+        FAPI_TRY(threads_running(i_target, i_threads, o_rasStatusReg, l_running),
                  "p9_thread_control_start: unable to determine if threads are running. threads: 0x%x",
                  i_threads);
 
@@ -408,13 +528,15 @@ fapi_try_exit:
 /// @param[in] i_target core target
 /// @param[in] i_threads normal core thread bitset (0b0000..0b1111)
 /// @param[in] i_warncheck convert pre/post checks errors to warnings
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
 /// @return FAPI2_RC_SUCCESS if operation was successful,
 ///          RC_P9_THREAD_CONTROL_STOP_FAIL if start command failed,
 ///         else error
 //--------------------------------------------------------------------------
 fapi2::ReturnCode p9_thread_control_stop(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck)
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg)
 {
     FAPI_DBG("p9_thread_control_stop : Initiating stop command to core PC logic for threads 0x%x",
              i_threads);
@@ -424,7 +546,7 @@ fapi2::ReturnCode p9_thread_control_stop(
     // TODO: Do we want to check to see if all threads are stopped and just bypass this if they are?
     {
         bool l_running = false;
-        FAPI_TRY(threads_running(i_target, i_threads, l_running),
+        FAPI_TRY(threads_running(i_target, i_threads, o_rasStatusReg, l_running),
                  "p9_thread_control_stop: unable to determine if threads are running. threads: 0x%x",
                  i_threads);
 
@@ -448,7 +570,7 @@ fapi2::ReturnCode p9_thread_control_stop(
     // Post-conditions check
     {
         bool l_stopped = false;
-        FAPI_TRY(threads_stopped(i_target, i_threads, l_stopped),
+        FAPI_TRY(threads_stopped(i_target, i_threads, o_rasStatusReg, l_stopped),
                  "p9_thread_control_stop: unable to determine if threads are stopped. threads: 0x%x",
                  i_threads);
 
@@ -474,13 +596,15 @@ fapi_try_exit:
 /// @param[in] i_target core target
 /// @param[in] i_threads normal core thread bitset (0b0000..0b1111)
 /// @param[in] i_warncheck convert pre/post checks errors to warnings
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer.
 /// @return FAPI2_RC_SUCCESS if operation was successful,
 ///          RC_P9_THREAD_CONTROL_STEP_FAIL if start command failed,
 ///         else error
 //--------------------------------------------------------------------------
 fapi2::ReturnCode p9_thread_control_step(
     const fapi2::Target<TARGET_TYPE_CORE>& i_target,
-    const uint8_t i_threads, const bool i_warncheck)
+    const uint8_t i_threads, const bool i_warncheck,
+    fapi2::buffer<uint64_t>& o_rasStatusReg)
 {
     FAPI_DBG("p9_thread_control_stop : Initiating step command to core PC logic for threads 0x%x",
              i_threads);
@@ -488,7 +612,7 @@ fapi2::ReturnCode p9_thread_control_step(
     // Preconditions
     {
         bool l_step_ready = false;
-        FAPI_TRY(threads_step_ready(i_target, i_threads, l_step_ready),
+        FAPI_TRY(threads_step_ready(i_target, i_threads, o_rasStatusReg, l_step_ready),
                  "p9_thread_control_step: unable to determine if threads are ready to step. threads: 0x%x",
                  i_threads);
 
@@ -533,7 +657,7 @@ fapi2::ReturnCode p9_thread_control_step(
         do
         {
             FAPI_DBG("polling for step done. governor: %d", l_governor);
-            FAPI_TRY(threads_step_done(i_target, i_threads, l_step_done),
+            FAPI_TRY(threads_step_done(i_target, i_threads, o_rasStatusReg, l_step_done),
                      "p9_thread_control_step: thread step issued but something went wrong polling for step_done for threads 0x%x",
                      i_threads);
         }
@@ -571,4 +695,3 @@ fapi2::ReturnCode p9_thread_control_step(
 fapi_try_exit:
     return fapi2::current_err;
 }
-
diff --git a/src/import/chips/p9/procedures/hwp/core/p9_thread_control.H b/src/import/chips/p9/procedures/hwp/core/p9_thread_control.H
index cf7c61562..c976f692b 100644..100755
--- a/src/import/chips/p9/procedures/hwp/core/p9_thread_control.H
+++ b/src/import/chips/p9/procedures/hwp/core/p9_thread_control.H
@@ -17,10 +17,6 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 //------------------------------------------------------------------------------
-// *! (C) Copyright International Business Machines Corp. 2012
-// *! All Rights Reserved -- Property of IBM
-// *! *** IBM Confidential ***
-//------------------------------------------------------------------------------
 // *! TITLE       : p9_thread_control
 // *! DESCRIPTION : Core Thread start/stop/step/query/activate operations
 // *!               Use to start (start or sreset) thread instruction execution,
@@ -29,7 +25,7 @@
 //------------------------------------------------------------------------------
 
 // *HWP HWP Owner: Nick Klazynski <dyem@us.ibm.com>
-// *HWP FW Owner:  Brian Silver <bsilver@us.ibm.com>
+// *HWP FW Owner:  Thi Tran <thi@us.ibm.com>
 // *HWP Team:  Quad
 // *HWP Level: 2
 // *HWP Consumed by: FSP:HB:HS
@@ -46,6 +42,21 @@
 extern "C"
 {
 
+//------------------------------------------------------------------------------
+// Constants
+//------------------------------------------------------------------------------
+    const uint8_t MAX_NUM_OF_THREADS = 4;
+
+// THREAD_STATE bit definitions
+    const uint64_t THREAD_STATE_RUNNING            = 0x8000000000000000ULL;
+    const uint64_t THREAD_STATE_STOP               = 0x4000000000000000ULL;
+    const uint64_t THREAD_STATE_QUIESCED           = 0x2000000000000000ULL;
+    const uint64_t THREAD_STATE_MAINT              = 0x1000000000000000ULL;
+    const uint64_t THREAD_STATE_ICT_EMPTY          = 0x0800000000000000ULL;
+    const uint64_t THREAD_STATE_LSU_QUIESCED       = 0x0400000000000000ULL;
+    const uint64_t THREAD_STATE_ISTEP_SUCCESS      = 0x0200000000000000ULL;
+    const uint64_t THREAD_STATE_ISTEP_READY        = 0x0100000000000000ULL;
+
 // A macro to wrap the warning check boiler plate
 // If the action failed and i_warncheck is set add a trace and continue anyway
 #define PTC_ASSERT_WARN( __conditional__, __warning__, __ffdc__, ... )  \
@@ -74,6 +85,7 @@ extern "C"
         PTC_CMD_STEP              = 1,
         PTC_CMD_START             = 2,
         PTC_CMD_STOP              = 3,
+        PTC_CMD_QUERY             = 4,
     };
 
     enum ThreadRasStatus
@@ -81,7 +93,7 @@ extern "C"
         CORE_MAINT_MODE = 0,
         THREAD_QUIESCED = 1,
         ICT_EMPTY       = 2,
-        ISU_QUIESCED    = 3,
+        LSU_QUIESCED    = 3,
         STEP_SUCCESS    = 4,
     };
 
@@ -105,7 +117,8 @@ extern "C"
 /// function pointer typedef definition for HWP call support
     typedef fapi2::ReturnCode (*p9_thread_control_FP_t) (
         const fapi2::Target<fapi2::TARGET_TYPE_CORE>&,
-        const uint8_t, const ThreadCommands, const bool);
+        const uint8_t, const ThreadCommands, const bool,
+        fapi2::buffer<uint64_t>&, uint64_t&);
 
 //--------------------------------------------------------------------------
 /// @brief p9_thread_control: utility subroutine to control thread state
@@ -126,11 +139,17 @@ extern "C"
 //        const uint8_t i_threads, const ThreadCommands i_command,
 //        const bool i_warncheck);
 
-
 //--------------------------------------------------------------------------
 /// @brief p9_thread_control: utility subroutine to control thread state
+///
 /// @param[in] i_target core target
-/// @param[in] i_thread normal core thread bitset (0b0000..0b1111)
+/// @param[in] i_threads     Desired thread bit set, multiple thread settings
+///                          are allowed.
+///                             0b0000         No thread (No-op)
+///                             0b1000         Thread 0
+///                             0b0100         Thread 1
+///                             0b0010         Thread 2
+///                             0b0001         Thread 3
 /// @param[in] i_command  one of
 ///               PTC_CMD_SRESET => initiate sreset thread command
 ///               PTC_CMD_START  => initiate start thread command
@@ -138,13 +157,21 @@ extern "C"
 ///               PTC_CMD_STEP   => initiate step thread command
 ///               PTC_CMD_QUERY  => query and return thread state
 /// @param[in] i_warncheck convert pre/post checks errors to warnings
+/// @param[out] o_rasStatusReg  Complete RAS status reg 64-bit buffer after
+///                             executing command.
+/// @param[out] o_state Thread state, only valid for PTC_CMD_QUERY command.
+///                    See p9_thread_control.H for THREAD_STATE bit definitions.
+///
 /// @return FAPI_RC_SUCCESS if operation was successful,
 ///         function-specific fail codes (see function definitions),
 ///         else error
 //--------------------------------------------------------------------------
     fapi2::ReturnCode p9_thread_control(const fapi2::Target<fapi2::TARGET_TYPE_CORE>& i_target,
                                         const uint8_t i_threads, const ThreadCommands i_command,
-                                        const bool i_warncheck);
-#endif // _P9_THREAD_CONTROL_H_
+                                        const bool i_warncheck,
+                                        fapi2::buffer<uint64_t>& o_rasStatusReg,
+                                        uint64_t& o_state);
 
 } // extern
+
+#endif // _P9_THREAD_CONTROL_H_
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_thread_control_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_thread_control_errors.xml
index 78056129f..af4d8c25a 100644
--- a/src/import/chips/p9/procedures/xml/error_info/p9_thread_control_errors.xml
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_thread_control_errors.xml
@@ -7,7 +7,7 @@
 <!--                                                                        -->
 <!-- EKB Project                                                            -->
 <!--                                                                        -->
-<!-- COPYRIGHT 2012,2015                                                    -->
+<!-- COPYRIGHT 2015,2016                                                    -->
 <!-- [+] International Business Machines Corp.                              -->
 <!--                                                                        -->
 <!--                                                                        -->