Atomically latch shutdown status and TI data together in shutdown

Consolidates updating shutdown status and the TI area into the same
atomic code region to prevent race conditions where one path invokes shutdown
first, but another one ends up racing ahead and replacing the correct TI data
with its own.

Change-Id: Ic1e185e9bb6f2fc1c512a18976b89b12d47eb823
CQ: SW440180
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/63760
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: ILYA SMIRNOV <ismirno@us.ibm.com>
Reviewed-by: Marshall J. Wilks <mjwilks@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 166 insertions, 75 deletions

diff --git a/src/usr/initservice/baseinitsvc/initservice.C b/src/usr/initservice/baseinitsvc/initservice.C
index f62e49264..a40317a30 100644
--- a/src/usr/initservice/baseinitsvc/initservice.C
+++ b/src/usr/initservice/baseinitsvc/initservice.C
@@ -615,6 +615,7 @@ InitService& InitService::getTheInstance( )
 
 InitService::InitService( ) :
     iv_shutdownInProgress(false),
+    iv_worst_status(false),
     iv_iStep( 0 ),
     iv_iSubStep( 0 )
 {
@@ -661,6 +662,49 @@ void InitService::registerBlock(void* i_vaddr, uint64_t i_size,
     mutex_unlock(&iv_registryMutex);
 }
 
+bool InitService::_setShutdownStatus(
+    const uint64_t i_status,
+    const uint32_t i_error_info)
+{
+    TRACFCOMP(g_trac_initsvc, "_setShutdownStatus(i_status=0x%016llX)",
+        i_status);
+
+    // Hostboot PLIDs always start with 0x9 (32-bit)
+    static const uint64_t PLID_MASK = 0x0000000090000000;
+    bool first=false;
+
+    // Ensure no one is manpulating the registry lists
+    mutex_lock(&iv_registryMutex);
+
+    // This already takes care of accepting only the first TI code, so
+    // just always call this
+    termWriteSRC(TI_SHUTDOWN,
+                 i_status,
+                 reinterpret_cast<uint64_t>(linkRegister()),
+                 i_error_info);
+
+    if (iv_shutdownInProgress)
+    {
+        // switch the failing status if an RC comes in after
+        //  a plid fail because we'd rather have the RC
+        if( ((iv_worst_status & 0x00000000F0000000) == PLID_MASK)
+            && ((i_status & 0x00000000F0000000) != PLID_MASK)
+            && (i_status > SHUTDOWN_STATUS_GOOD) )
+        {
+            iv_worst_status = i_status;
+        }
+    }
+    else
+    {
+        first=true;
+        iv_worst_status = i_status;
+        iv_shutdownInProgress = true;
+    }
+
+    mutex_unlock(&iv_registryMutex);
+
+    return first;
+}
 
 void doShutdown(uint64_t i_status,
                 bool i_inBackground,
@@ -670,11 +714,27 @@ void doShutdown(uint64_t i_status,
                 uint64_t i_masterHBInstance,
                 uint32_t i_error_info)
 {
-    termWriteSRC(TI_SHUTDOWN,
-                 i_status,
-                 reinterpret_cast<uint64_t>(linkRegister()),
-                 i_error_info);
+    (void)InitService::doShutdown(
+                           i_status,
+                           i_inBackground,
+                           i_payload_base,
+                           i_payload_entry,
+                           i_payload_data,
+                           i_masterHBInstance,
+                           i_error_info);
+}
 
+void InitService::doShutdown(
+                      uint64_t i_status,
+                      bool     i_inBackground,
+                      uint64_t i_payload_base,
+                      uint64_t i_payload_entry,
+                      uint64_t i_payload_data,
+                      uint64_t i_masterHBInstance,
+                      uint32_t i_error_info)
+{
+    bool first = Singleton<InitService>::instance()._setShutdownStatus(
+                                                        i_status,i_error_info);
     class ShutdownExecute
     {
         public:
@@ -694,7 +754,7 @@ void doShutdown(uint64_t i_status,
 
             void execute()
             {
-                Singleton<InitService>::instance().doShutdown(status,
+                Singleton<InitService>::instance()._doShutdown(status,
                                                               payload_base,
                                                               payload_entry,
                                                               payload_data,
@@ -730,15 +790,30 @@ void doShutdown(uint64_t i_status,
     ShutdownExecute* s = new ShutdownExecute(i_status, i_payload_base,
                                              i_payload_entry, i_payload_data,
                                              i_masterHBInstance, i_error_info);
-
-    if (i_inBackground)
+    if(first)
     {
-        s->startThread();
+        if (i_inBackground)
+        {
+            s->startThread();
+        }
+        else
+        {
+            s->execute();
+            while(1)
+            {
+                nanosleep(1,0);
+            }
+        }
     }
     else
     {
-        s->execute();
-        while(1) nanosleep(1,0);
+        if(!i_inBackground)
+        {
+            while(1)
+            {
+                nanosleep(1,0);
+            }
+        }
     }
 }
 
@@ -760,18 +835,17 @@ class isPriority
         uint32_t iv_priority;
 };
 
-void InitService::doShutdown(uint64_t i_status,
-                             uint64_t i_payload_base,
-                             uint64_t i_payload_entry,
-                             uint64_t i_payload_data,
-                             uint64_t i_masterHBInstance,
-                             uint32_t i_error_info)
+void InitService::_doShutdown(uint64_t i_status,
+                              uint64_t i_payload_base,
+                              uint64_t i_payload_entry,
+                              uint64_t i_payload_data,
+                              uint64_t i_masterHBInstance,
+                              uint32_t i_error_info)
 {
     int l_rc = 0;
     errlHndl_t l_err = NULL;
-    static volatile uint64_t worst_status = 0;
 
-    TRACFCOMP(g_trac_initsvc, "doShutdown(i_status=%.16X)",i_status);
+    TRACFCOMP(g_trac_initsvc, "_doShutdown(i_status=%.16X)",i_status);
 #ifdef CONFIG_CONSOLE
     // check if console msg not needed or already displayed by caller
     if ((SHUTDOWN_STATUS_GOOD != i_status) &&
@@ -788,31 +862,7 @@ void InitService::doShutdown(uint64_t i_status,
     }
 #endif
 
-    // Hostboot PLIDs always start with 0x9 (32-bit)
-    static const uint64_t PLID_MASK = 0x0000000090000000;
-
-    // Ensure no one is manpulating the registry lists and that only one
-    // thread actually executes the shutdown path.
-    mutex_lock(&iv_registryMutex);
-
-    if (iv_shutdownInProgress)
-    {
-        // switch the failing status if an RC comes in after
-        //  a plid fail because we'd rather have the RC
-        if( ((worst_status & 0x00000000F0000000) == PLID_MASK)
-            && ((i_status & 0x00000000F0000000) != PLID_MASK)
-            && (i_status > SHUTDOWN_STATUS_GOOD) )
-        {
-            worst_status = i_status;
-        }
-        mutex_unlock(&iv_registryMutex);
-        return;
-    }
-    worst_status = i_status; //first thread in is the default
-    iv_shutdownInProgress = true;
-    mutex_unlock(&iv_registryMutex);
-
-    TRACFCOMP(g_trac_initsvc, "doShutdown> status=%.16X",worst_status);
+    TRACFCOMP(g_trac_initsvc, "_doShutdown> status=%.16X",iv_worst_status);
 
     // sort the queue by priority before sending..
     std::sort( iv_regMsgQ.begin(), iv_regMsgQ.end());
@@ -828,7 +878,7 @@ void InitService::doShutdown(uint64_t i_status,
     {
             TRACFCOMP(g_trac_initsvc,"notify priority=0x%x, queue=0x%x", i->msgPriority, i->msgQ );
             l_msg->type = i->msgType;
-            l_msg->data[0] = worst_status;
+            l_msg->data[0] = iv_worst_status;
             (void)msg_sendrecv(i->msgQ,l_msg);
     }
 
@@ -878,7 +928,7 @@ void InitService::doShutdown(uint64_t i_status,
     {
             TRACDCOMP(g_trac_initsvc,"notify priority=0x%x, queue=0x%x", i->msgPriority, i->msgQ );
             l_msg->type = i->msgType;
-            l_msg->data[0] = worst_status;
+            l_msg->data[0] = iv_worst_status;
             (void)msg_sendrecv(i->msgQ,l_msg);
     }
     msg_free(l_msg);
@@ -891,17 +941,17 @@ void InitService::doShutdown(uint64_t i_status,
     task_yield();
     nanosleep(0,TEN_CTX_SWITCHES_NS);
 
-    TRACFCOMP(g_trac_initsvc, "doShutdown> Final status=%.16X",worst_status);
+    TRACFCOMP(g_trac_initsvc, "_doShutdown> Final status=%.16X",iv_worst_status);
     MAGIC_INST_PRINT_ISTEP(21,4);
 
     // Update the HB TI area with the worst status.
     termWriteSRC(TI_SHUTDOWN,
-                 worst_status,
+                 iv_worst_status,
                  reinterpret_cast<uint64_t>(linkRegister()),
                  i_error_info,
                  true); // Force write
 
-    shutdown(worst_status,
+    shutdown(iv_worst_status,
              i_payload_base,
              i_payload_entry,
              i_payload_data,
diff --git a/src/usr/initservice/baseinitsvc/initservice.H b/src/usr/initservice/baseinitsvc/initservice.H
index 18e76ef49..c3b71584a 100644
--- a/src/usr/initservice/baseinitsvc/initservice.H
+++ b/src/usr/initservice/baseinitsvc/initservice.H
@@ -215,32 +215,6 @@ public:
     bool unregisterShutdownEvent(msg_q_t i_msgQ);
 
     /**
-     * @brief Perform necessary shut down steps.
-     *
-     * @param[in] i_status - Shutdown status to be passed along on shutdown
-     * @param[in] i_payload_base - The base address (target HRMOR) of the
-     *                             payload.
-     * @param[in] i_payload_entry - The offset from base address of the
-     *                              payload entry-point.
-     * @param[in] i_payload_entry - HRMOR adjusted address of any payload data
-     *                              placed in r3
-     * @param[in] i_masterHBInstance - master hostboot instance number (node)
-     *                                 Needed when starting payload on a
-     *                                 multi-node system.
-     * @param[in] i_error_info - Additional error data to be included in TI data
-     *
-     * @return Nothing
-     * @note This calls registered services to notify them of shutdown and it
-     * flushes the virtual memory.
-     */
-    void doShutdown ( uint64_t i_status,
-                      uint64_t i_payload_base = 0,
-                      uint64_t i_payload_entry = 0,
-                      uint64_t i_payload_data = 0,
-                      uint64_t i_masterHBInstance = 0xffffffffffffffffull,
-                      uint32_t i_error_info = 0);
-
-    /**
      * @brief Save Istep Step and Substep for use by error logging
      * @param[in] i_step, Istep Step
      * @param[in] i_subStep, Istep subStep
@@ -258,6 +232,20 @@ public:
     void GetIstepData( uint8_t & o_step,
                        uint8_t & o_subStep );
 
+    /**
+     *  @brief Shut down Hostboot in a controlled way
+     *
+     *  @note: See the API documentation @
+     *      src/include/usr/initservice/initserviceif.H
+     */
+    static void doShutdown (
+                     uint64_t i_status,
+                     bool     i_inBackground = false,
+                     uint64_t i_payload_base = 0,
+                     uint64_t i_payload_entry = 0,
+                     uint64_t i_payload_data = 0,
+                     uint64_t i_masterHBInstance = THIS_NODE_NO_PAYLOAD,
+                     uint32_t i_error_info = 0);
 
 protected:
 
@@ -271,7 +259,6 @@ protected:
      */
     ~InitService();
 
-
 private:
     /**
      *  Disable copy constructor and assignment operator
@@ -279,6 +266,57 @@ private:
     InitService(const InitService& i_right);
     InitService& operator=(const InitService& i_right);
 
+    /**
+     * @brief Atomically sets the shutdown status and TI data associated with
+     *     a shutdown request
+     *
+     * @par Detailed Description
+     *     Atomically sets the shutdown status and TI data associated with
+     *     a shutdown request.  Any number of callers may initiate parallel
+     *     shutdown requests, however this interface ensures that the first
+     *     caller atomically updates the initial TI area and shutdown status.
+     *     Subsequent callers will only be able to lock in a new status (TI
+     *     area remains untouched) if it is a non-success reason code and the
+     *     initial status was a PLID.
+     *
+     * @warning Should only be called by INITSERVICE::doShutdown
+     *
+     * @param[in] i_status     Shutdown status to be passed to primary shutdown
+     *     handler
+     * @param[in] i_error_info Additional error data to be included in TI data
+     *
+     * @return bool Boolean indicating whether this was the first shutdown
+     *     to be requested
+     */
+    bool _setShutdownStatus(
+        uint64_t i_status,
+        uint32_t i_error_info);
+
+    /**
+     * @brief Perform necessary shut down steps.
+     *
+     * @param[in] i_status - Shutdown status to be passed along on shutdown
+     * @param[in] i_payload_base - The base address (target HRMOR) of the
+     *                             payload.
+     * @param[in] i_payload_entry - The offset from base address of the
+     *                              payload entry-point.
+     * @param[in] i_payload_entry - HRMOR adjusted address of any payload data
+     *                              placed in r3
+     * @param[in] i_masterHBInstance - master hostboot instance number (node)
+     *                                 Needed when starting payload on a
+     *                                 multi-node system.
+     * @param[in] i_error_info - Additional error data to be included in TI data
+     *
+     * @return Nothing
+     * @note This calls registered services to notify them of shutdown and it
+     * flushes the virtual memory.
+     */
+    void _doShutdown (uint64_t i_status,
+                      uint64_t i_payload_base = 0,
+                      uint64_t i_payload_entry = 0,
+                      uint64_t i_payload_data = 0,
+                      uint64_t i_masterHBInstance = 0xffffffffffffffffull,
+                      uint32_t i_error_info = 0);
 
     /**
      * Check and load module associated with this task or function
@@ -325,6 +363,9 @@ private:
     mutex_t iv_registryMutex;
     bool iv_shutdownInProgress;
 
+    // Worst shutdown status accumulated during shutdown processing
+    uint64_t iv_worst_status;
+
     uint8_t iv_iStep;     // shadow of current istep / substep
     uint8_t iv_iSubStep;  //  for error logs