Handles FSP reset-reload in HBRT messages

For every instance where a firmware_request is made, it is possible
to get an error if the FSP is doing a reset/reload.  Now, if an
error returned from the firmware_request call indicates that the
FSP is doing a reset/reload, the firmware_request call will be made
again. Also refactored the code.  Consolidated the calls to
firmware_request to another file to facilitate the adding/updating
of messages.

Change-Id: I5be440927ab255c55b40a0a0ed26f786d31a9317
RTC:182606
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/50505
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

3 files changed, 646 insertions, 190 deletions

diff --git a/src/usr/util/runtime/makefile b/src/usr/util/runtime/makefile
index ff5509e5e..b926c9209 100644
--- a/src/usr/util/runtime/makefile
+++ b/src/usr/util/runtime/makefile
@@ -53,6 +53,7 @@ OBJS += utilrsvdmem.o
 OBJS += utilxipimage.o
 OBJS += utilcommonattr.o
 OBJS += rt_fwnotify.o
+OBJS += rt_fwreq_helper.o
 
 SUBDIRS += test.d
 
diff --git a/src/usr/util/runtime/rt_fwnotify.C b/src/usr/util/runtime/rt_fwnotify.C
index 9441519ab..fd6037512 100644
--- a/src/usr/util/runtime/rt_fwnotify.C
+++ b/src/usr/util/runtime/rt_fwnotify.C
@@ -22,35 +22,14 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
-/* This is an automatically generated prolog.                             */
-/*                                                                        */
-/* $Source: src/usr/util/runtime/rt_fwnotify.C $                          */
-/*                                                                        */
-/* OpenPOWER HostBoot Project                                             */
-/*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
-/* [+] International Business Machines Corp.                              */
-/*                                                                        */
-/*                                                                        */
-/* Licensed under the Apache License, Version 2.0 (the "License");        */
-/* you may not use this file except in compliance with the License.       */
-/* You may obtain a copy of the License at                                */
-/*                                                                        */
-/*     http://www.apache.org/licenses/LICENSE-2.0                         */
-/*                                                                        */
-/* Unless required by applicable law or agreed to in writing, software    */
-/* distributed under the License is distributed on an "AS IS" BASIS,      */
-/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
-/* implied. See the License for the specific language governing           */
-/* permissions and limitations under the License.                         */
-/*                                                                        */
-/* IBM_PROLOG_END_TAG                                                     */
 
 #include <sbeio/sbe_retry_handler.H>       // SbeRetryHandler
-#include <runtime/interface.h>             // firmware_notify
+#include <util/runtime/rt_fwreq_helper.H>       // firmware_request_helper
+#include <runtime/interface.h>             // g_hostInterfaces
 #include <runtime/runtime_reasoncodes.H>   // MOD_RT_FIRMWARE_NOTIFY, etc
 #include <errl/errlentry.H>                // ErrlEntry
 #include <errl/errlmanager.H>              // errlCommit
+#include <errl/hberrltypes.H>              // TWO_UINT32_TO_UINT64
 #include <targeting/common/target.H>       // TargetHandle_t, getTargetFromHuid
 
 using namespace TARGETING;
@@ -59,36 +38,39 @@ using namespace ERRORLOG;
 using namespace MBOX;
 using namespace SBEIO;
 
-trace_desc_t* g_trac_hwsv = nullptr;
-TRAC_INIT(&g_trac_hwsv, "HWSV_TRACE", 4*KILOBYTE);
+// Trace definition
+extern trace_desc_t* g_trac_runtime;
 
 /**
-  * @brief The lower and upper bounds for the sequence ID.
-  **/
-const uint16_t GFMM_SEQ_ID_MIN = 0x0000;
-const uint16_t GFMM_SEQ_ID_MAX =  0x7FFF;
+ * @brief The lower and upper bounds for the sequence ID.
+ **/
+const uint16_t SEQ_ID_MIN = 0x0000;
+const uint16_t SEQ_ID_MAX =  0x7FFF;
 
 /**
-  * @brief Set the sequence ID to the minimum value
-  **/
-uint16_t SeqId_t::GFMM_SEQ_ID = GFMM_SEQ_ID_MIN;
+ * @brief Set the sequence ID to the minimum value
+ **/
+uint16_t SeqId_t::SEQ_ID = SEQ_ID_MIN;
 
 /**
- *  @brief Gets the next sequence ID.
- *  @return The next sequence ID value within it's lower and upper bound
+ * @brief Gets the next sequence ID.
+ * @return The next sequence ID value within its lower and upper bound
+ * @note This code is thread safe, no need for a mutex around this because
+ *       HBRT never runs in multi-thread.
+ *
  **/
 uint16_t SeqId_t::getNextSeqId()
 {
-    if (SeqId_t::GFMM_SEQ_ID < GFMM_SEQ_ID_MAX)
+    if (SeqId_t::SEQ_ID < SEQ_ID_MAX)
     {
-        ++SeqId_t::GFMM_SEQ_ID;
+        ++SeqId_t::SEQ_ID;
     }
     else
     {
-        SeqId_t::GFMM_SEQ_ID = GFMM_SEQ_ID_MIN;
+        SeqId_t::SEQ_ID = SEQ_ID_MIN;
     }
 
-    return SeqId_t::GFMM_SEQ_ID;
+    return SeqId_t::SEQ_ID;
 }
 
 /**
@@ -97,7 +79,7 @@ uint16_t SeqId_t::getNextSeqId()
  **/
 uint16_t SeqId_t::getCurrentSeqId()
 {
-  return SeqId_t::GFMM_SEQ_ID;
+  return SeqId_t::SEQ_ID;
 }
 
 /**
@@ -108,34 +90,34 @@ uint16_t SeqId_t::getCurrentSeqId()
  **/
 void sbeAttemptRecovery(uint64_t i_data)
 {
+   // Create a useful struct to get to the data
+   // The data is expected to be a plid (in the first 4 bytes)
+   // followed by a HUID (in the last 4 bytes).
+   HbrtFspData_t *l_hbrtFspData = reinterpret_cast<HbrtFspData_t*>(&i_data);
+
+   TRACFCOMP(g_trac_runtime, ENTER_MRK"sbeAttemptRecovery: plid:0x%X, "
+             "HUID:0x%X", l_hbrtFspData->plid, l_hbrtFspData->userData);
+
     errlHndl_t l_err = nullptr;
 
     do
     {
-        // Create a useful structure to get to the data
-        // The data is expected to be a plid (in the first 4 bytes)
-        // followed by a HUID (in the last 4 bytes).
-        struct sbeErrorData_t
-        {
-            uint32_t plid;
-            uint32_t huid;
-        } PACKED ;
-
-        sbeErrorData_t *l_sbeErrorDataReq = (sbeErrorData_t*)(&i_data);
-
         // Extract the target from the given HUID
         TargetHandle_t l_target =
-                        Target::getTargetFromHuid(l_sbeErrorDataReq->huid);
+                        Target::getTargetFromHuid(l_hbrtFspData->userData);
 
         // If HUID invalid, log error and quit
         if (nullptr == l_target)
         {
-             TRACFCOMP( g_trac_hwsv, "firmware_notify: No target assoicated "
-                       "with HUID:0x%.8X", l_sbeErrorDataReq->huid);
+             TRACFCOMP(g_trac_runtime, ERR_MRK"sbeAttemptRecovery: "
+                       "No target associated with HUID:0x%.8X",
+                       l_hbrtFspData->userData);
+
             /*@
-             * @errortype    ERRL_SEV_PREDICTIVE
-             * @moduleid     RUNTIME::MOD_RT_FIRMWARE_NOTIFY
-             * @reasoncode   RUNTIME::RC_SBE_RT_INVALID_HUID
+             * @errortype
+             * @severity     ERRL_SEV_PREDICTIVE
+             * @moduleid     MOD_RT_FIRMWARE_NOTIFY
+             * @reasoncode   RC_SBE_RT_INVALID_HUID
              * @userdata1    HUID of target
              * @userdata2    HWSV error log id (plid)
              * @devdesc      SBE error recovery attempt failed
@@ -143,8 +125,8 @@ void sbeAttemptRecovery(uint64_t i_data)
             l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
                                    MOD_RT_FIRMWARE_NOTIFY,
                                    RC_SBE_RT_INVALID_HUID,
-                                   l_sbeErrorDataReq->huid,
-                                   l_sbeErrorDataReq->plid,
+                                   l_hbrtFspData->userData,
+                                   l_hbrtFspData->plid,
                                    true);
             break;
         }
@@ -163,13 +145,14 @@ void sbeAttemptRecovery(uint64_t i_data)
         if (nullptr == g_hostInterfaces ||
             nullptr == g_hostInterfaces->firmware_request)
         {
-             TRACFCOMP( g_trac_hwsv, "firmware_notify: Hypervisor "
-                       "firmware_request interface not linked");
+             TRACFCOMP(g_trac_runtime, ERR_MRK"sbeAttemptRecovery: "
+                       "Hypervisor firmware_request interface not linked");
 
             /*@
-             * @errortype    ERRL_SEV_PREDICTIVE
-             * @moduleid     RUNTIME::MOD_RT_FIRMWARE_NOTIFY
-             * @reasoncode   RUNTIME::RC_FW_REQUEST_RT_NULL_PTR
+             * @errortype
+             * @severity     ERRL_SEV_PREDICTIVE
+             * @moduleid     MOD_RT_FIRMWARE_NOTIFY
+             * @reasoncode   RC_FW_REQUEST_RT_NULL_PTR
              * @userdata1    HUID of target
              * @userdata2    HWSV error log id (plid)
              * @devdesc      SBE error recovery attempt failed
@@ -177,105 +160,60 @@ void sbeAttemptRecovery(uint64_t i_data)
             l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
                                    MOD_RT_FIRMWARE_NOTIFY,
                                    RC_FW_REQUEST_RT_NULL_PTR,
-                                   l_sbeErrorDataReq->huid,
-                                   l_sbeErrorDataReq->plid,
+                                   l_hbrtFspData->userData,
+                                   l_hbrtFspData->plid,
                                    true);
 
-
            break;
         }
 
-        // Create the firmware_request request structure to send data
+        // Create the firmware_request request struct to send data
         hostInterfaces::hbrt_fw_msg l_req_fw_msg;
-        memset(&l_req_fw_msg, 0, sizeof(l_req_fw_msg));
+        uint64_t l_req_fw_msg_size = sizeof(l_req_fw_msg);
 
-        // Set the data for the request
-        l_req_fw_msg.io_type = hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ;
+        // Initialize the firmware_request request struct
         l_req_fw_msg.generic_msg.initialize();
-        l_req_fw_msg.generic_msg.msgq = GFMM_MSG_SBE_ERROR;
-        l_req_fw_msg.generic_msg.__req = GFMM_REQUEST;
-        l_req_fw_msg.generic_msg.__onlyError = GFMM_NOT_ERROR_ONLY;
+
+        // Populate the firmware_request request struct with given data
+        l_req_fw_msg.io_type = hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ;
+        l_req_fw_msg.generic_msg.msgq = GenericFspMboxMessage_t::MSG_SBE_ERROR;
+        l_req_fw_msg.generic_msg.__req = GenericFspMboxMessage_t::REQUEST;
         l_req_fw_msg.generic_msg.data = i_data;
 
+        // Set msgType based on recovery success or failure
         if (l_recoverySuccessful)
         {
-            l_req_fw_msg.generic_msg.msgType= GFMM_MSG_SBE_RECOVERY_SUCCESS;
+            l_req_fw_msg.generic_msg.msgType =
+                             GenericFspMboxMessage_t::MSG_SBE_RECOVERY_SUCCESS;
         }
         else
         {
-            l_req_fw_msg.generic_msg.msgType = GFMM_MSG_SBE_RECOVERY_FAILED;
+            l_req_fw_msg.generic_msg.msgType =
+                              GenericFspMboxMessage_t::MSG_SBE_RECOVERY_FAILED;
         }
 
-        // Create the firmware_request response structure to receive data
+        // Create the firmware_request response struct to receive data
+        // NOTE: For messages to the FSP the response size must match
+        // the request size
+        // No need to check for expected response size > request
+        // size because they use the same base structure
         hostInterfaces::hbrt_fw_msg l_resp_fw_msg;
-        size_t l_resp_fw_msg_size = sizeof(l_resp_fw_msg);
+        uint64_t l_resp_fw_msg_size = sizeof(l_resp_fw_msg);
         memset(&l_resp_fw_msg, 0, l_resp_fw_msg_size);
 
-        size_t rc = g_hostInterfaces->firmware_request(sizeof(l_req_fw_msg),
-                                                       &l_req_fw_msg,
-                                                       &l_resp_fw_msg_size,
-                                                       &l_resp_fw_msg);
-
-        // Error log id
-        uint32_t l_errPlid(0);
+        // Trace out the request structure
+        TRACFBIN( g_trac_runtime, INFO_MRK"Sending firmware_request",
+                  &l_req_fw_msg,
+                  l_req_fw_msg_size);
 
-        sbeErrorData_t *l_sbeErrorDataResp =
-                   (sbeErrorData_t*)&(l_resp_fw_msg.generic_msg.data);
+        // Make the firmware_request call
+        l_err = firmware_request_helper(l_req_fw_msg_size,
+                                        &l_req_fw_msg,
+                                        &l_resp_fw_msg_size,
+                                        &l_resp_fw_msg);
 
-        // Capture the error log ID if any
-        // The return code (rc) may return OK, but there still may be an issue
-        // with the HWSV code on the FSP.
-        if ((hostInterfaces::HBRT_FW_MSG_HBRT_FSP_RESP
-                                                  == l_resp_fw_msg.io_type) &&
-            (GFMM_ERROR_ONLY == l_resp_fw_msg.generic_msg.__onlyError) &&
-            (0 != l_sbeErrorDataResp->plid) )
+        if (l_err)
         {
-            l_errPlid = l_sbeErrorDataResp->plid;
-        }
-
-        // Gather up the error data and create an error log out of it
-        if (rc || l_errPlid)
-        {
-            /*@
-             * @errortype        ERRL_SEV_PREDICTIVE
-             * @moduleid         RUNTIME::MOD_RT_FIRMWARE_NOTIFY
-             * @reasoncode       RUNTIME::RC_SBE_RT_RECOVERY_ERR
-             * @userdata1[0:31]  Firmware Request return code
-             * @userdata1[32:63] HWSV error log id (plid)
-             * @userdata2[0:31]  MBOX message type
-             * @userdata2[32:63] Message Tyepe
-             * @devdesc          SBE error recovery attempt failed
-             */
-            l_err = new ERRORLOG::ErrlEntry(
-                                   ERRL_SEV_PREDICTIVE,
-                                   MOD_RT_FIRMWARE_NOTIFY,
-                                   RC_SBE_RT_RECOVERY_ERR,
-                                   TWO_UINT32_TO_UINT64(rc, l_errPlid),
-                                   TWO_UINT32_TO_UINT64(
-                                   l_resp_fw_msg.generic_msg.msgq,
-                                   l_resp_fw_msg.generic_msg.msgType),
-                                   true);
-
-            l_err->addFFDC( RUNTIME_COMP_ID,
-                            &l_resp_fw_msg,
-                            l_resp_fw_msg_size,
-                            0, 0, false );
-
-            if (sizeof(l_req_fw_msg) > 0)
-            {
-                l_err->addFFDC( RUNTIME_COMP_ID,
-                                &l_req_fw_msg,
-                                sizeof(l_req_fw_msg),
-                                0, 0, false );
-            }
-
-            l_err->collectTrace( "SBE", 256);
-
-            if (l_errPlid)
-            {
-                l_err->plid(l_errPlid);
-            }
-
             break;
         }
     } while(0);
@@ -285,84 +223,146 @@ void sbeAttemptRecovery(uint64_t i_data)
        //Commit the error if it exists
        errlCommit(l_err, RUNTIME_COMP_ID);
     }
+
+    TRACFCOMP(g_trac_runtime, EXIT_MRK"sbeAttemptRecovery");
 }
 
 /**
- * @brief Receive an async notification from firmware
- * @param[in] i_len   length of notification data
- * @param[in] i_data  notification data
- * @platform FSP, OpenPOWER
+ * @see  src/include/runtime/interface.h for definition of call
+ *
  */
 void firmware_notify( uint64_t i_len, void *i_data )
 {
+   TRACFCOMP(g_trac_runtime, ENTER_MRK"firmware_notify: "
+             "i_len:%d", i_len );
+
+   TRACFBIN(g_trac_runtime, "firmware_notify: i_data", i_data, i_len);
+
     errlHndl_t l_err = nullptr;
 
+    // Flag to detect an invlaid/unknown/not used message
+    bool l_badMessage = false;
+
+    // Capture the unique message data associated with errant message
+    uint64_t l_userData1(0), l_userData2(0);
+
     do
     {
         // make sure the length of the data is not less than the
-        // structure we are expecting to receive
-        if (i_len < sizeof(GenericFspMboxMessage_t))
+        // data necessary to determine type of message
+        if (i_len < hostInterfaces::HBRT_FW_MSG_BASE_SIZE)
         {
-            TRACFCOMP( g_trac_hwsv,
-               "firmware_notify: Received a non GenericFspMboxMessage "
-               "data stream" );
+           l_badMessage = true;
 
-            /*@
-             * @errortype
-             * @moduleid     RUNTIME::MOD_RT_FIRMWARE_NOTIFY
-             * @reasoncode   RUNTIME::RC_FW_NOTIFY_RT_INVALID_MSG
-             * @userdata1    Unused
-             * @userdata2    Unused
-             * @devdesc      Error with Firmware Notify request
-             */
-            l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
-                                   MOD_RT_FIRMWARE_NOTIFY,
-                                   RC_FW_NOTIFY_RT_INVALID_MSG,
-                                   0, 0, true);
-            break;
-        }
+           TRACFCOMP(g_trac_runtime, ERR_MRK"firmware_notify: "
+               "Received a non hostInterfaces::hbrt_fw_msg data stream" );
 
-        // Cast the data to the structure we wish to parse
-        GenericFspMboxMessage_t* genericMsg =
-                       static_cast<GenericFspMboxMessage_t*>(i_data);
+           break;
+        }
 
-        // Do function based on message type (msgType)
-        switch(genericMsg->msgType)
+        // Cast the data to an hbrt_fw_msg to extract the input type
+        hostInterfaces::hbrt_fw_msg* l_hbrt_fw_msg =
+                       static_cast<hostInterfaces::hbrt_fw_msg*>(i_data);
+        switch (l_hbrt_fw_msg->io_type)
         {
-           case GFMM_MSG_SBE_ERROR: sbeAttemptRecovery(genericMsg->data);
-               break;
+
+           case hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ:
+           {
+
+              // Extract the message type
+              switch(l_hbrt_fw_msg->generic_msg.msgType)
+              {
+
+                 case GenericFspMboxMessage_t::MSG_SBE_ERROR:
+                      sbeAttemptRecovery(l_hbrt_fw_msg->generic_msg.data);
+
+                 break;
+
+                 default:
+                    {
+                       l_badMessage = true;
+
+                       TRACFCOMP(g_trac_runtime, ERR_MRK"firmware_notify: "
+                              "Unknown FSP message type:0x%.8X, "
+                              "message queue id:0x%.8X, seqNum:%d ",
+                              l_hbrt_fw_msg->generic_msg.msgq,
+                              l_hbrt_fw_msg->generic_msg.msgType,
+                              l_hbrt_fw_msg->generic_msg.seqnum);
+
+                       // Pack user data 1 with message input type and
+                       // firmware request message sequence number
+                       l_userData1 = TWO_UINT32_TO_UINT64(
+                                           l_hbrt_fw_msg->io_type,
+                                           l_hbrt_fw_msg->generic_msg.seqnum);
+
+                       // Pack user data 2 with message queue and message type
+                       l_userData2 = TWO_UINT32_TO_UINT64(
+                                            l_hbrt_fw_msg->generic_msg.msgq,
+                                            l_hbrt_fw_msg->generic_msg.msgType);
+                    }
+
+                 break;  // END default
+
+              }  // END switch(l_genericMsg->msgType)
+           } // END case HBRT_FW_MSG_HBRT_FSP_REQ:
+
+           break;
 
            default:
                {
-                   TRACFCOMP( g_trac_hwsv, "firmware_notify: Unknown "
-                              "message type:0x%.8X, message queue id:0x%.8X ",
-                              genericMsg->msgq, genericMsg->msgType);
-
-                   /*@
-                    * @errortype
-                    * @moduleid     RUNTIME::MOD_RT_FIRMWARE_NOTIFY
-                    * @reasoncode   RUNTIME::RC_FW_NOTIFY_RT_INVALID_MSG_TYPE
-                    * @userdata1    Message Queue ID
-                    * @userdata2    Message Type
-                    * @devdesc      Error with Firmware Notify request
-                    */
-                   l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
-                                          MOD_RT_FIRMWARE_NOTIFY,
-                                          RC_FW_NOTIFY_RT_INVALID_MSG_TYPE,
-                                          genericMsg->msgq,
-                                          genericMsg->msgType,
-                                          true);
-               }
+                  l_badMessage = true;
+
+                  TRACFCOMP(g_trac_runtime, ERR_MRK"firmware_notify: "
+                            "Unknown firmware request input type:0x%.8X ",
+                            l_hbrt_fw_msg->io_type);
+
+                  l_userData1 = l_hbrt_fw_msg->io_type;
+               }  // END default
+
                break;
-        };
+
+        };  // END switch (l_hbrt_fw_msg->io_type)
 
     } while(0);
 
+    if (l_badMessage)
+    {
+       /*@
+        * @errortype
+        * @severity     ERRL_SEV_PREDICTIVE
+        * @moduleid     MOD_RT_FIRMWARE_NOTIFY
+        * @reasoncode   RC_FW_NOTIFY_RT_INVALID_MSG
+        * @userdata1[0:31]  Firmware Request type
+        * @userdata1[32:63] Sequence number (FSP msg)
+        * @userdata2[0:31]  MBOX message type (FSP msg)
+        * @userdata2[32:63] Message Type (FSP msg)
+        * @devdesc      Error with Firmware Notify request
+        */
+       l_err = new ErrlEntry(ERRL_SEV_PREDICTIVE,
+                             MOD_RT_FIRMWARE_NOTIFY,
+                             RC_FW_NOTIFY_RT_INVALID_MSG,
+                             l_userData1,
+                             l_userData2,
+                             true);
+
+      if (i_len > 0)
+      {
+         l_err->addFFDC(RUNTIME_COMP_ID,
+                        i_data,
+                        i_len,
+                        0, 0, false );
+      }
+
+      l_err->collectTrace( "FW_REQ", 256);
+    }
+
     if (l_err)
     {
        //Commit the error if it exists
        errlCommit(l_err, RUNTIME_COMP_ID);
     }
+
+   TRACFCOMP(g_trac_runtime, EXIT_MRK"firmware_notify");
 };
 
 struct registerFwNotify
diff --git a/src/usr/util/runtime/rt_fwreq_helper.C b/src/usr/util/runtime/rt_fwreq_helper.C
new file mode 100644
index 000000000..16c9de777
--- /dev/null
+++ b/src/usr/util/runtime/rt_fwreq_helper.C
@@ -0,0 +1,455 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/util/runtime/rt_fwreq_helper.C $                      */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2017                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+
+#include <util/runtime/rt_fwreq_helper.H>
+#include <runtime/interface.h>            // hostInterfaces
+#include <runtime/runtime_reasoncodes.H>  // MOD_RT_FIRMWARE_REQUEST, etc
+#include <errl/errlmanager.H>             // errlCommit
+
+using namespace ERRORLOG;
+using namespace RUNTIME;
+using namespace TRACE;
+
+// Trace definition
+trace_desc_t *g_trac_runtime = nullptr;
+TRAC_INIT(&g_trac_runtime, RUNTIME_COMP_NAME, KILOBYTE);
+
+
+/*****************************************************************************/
+// firmware_request_helper
+/*****************************************************************************/
+errlHndl_t firmware_request_helper(uint64_t i_reqLen,   void *i_req,
+                                   uint64_t* o_respLen, void *o_resp)
+{
+   errlHndl_t l_err = nullptr;
+
+   // Cast incoming parameters to useful structs
+   hostInterfaces::hbrt_fw_msg *l_req_fw_msg =
+                    reinterpret_cast<hostInterfaces::hbrt_fw_msg *>(i_req);
+   hostInterfaces::hbrt_fw_msg *l_resp_fw_msg =
+                    reinterpret_cast<hostInterfaces::hbrt_fw_msg *>(o_resp);
+
+   // Capture the Reset/Reload error log if it happens
+   bool l_isResetReloadErr = false;
+
+   // Retry the firmware_request call if the call is made while the
+   // FSP is doing a reset/reload.
+   // If there is a reset/reload error create an error log and try again.
+   // If subsequent calls cause an error, commit the reset/reload error,
+   // and return the latest error back to the user.
+   // If there is an initial error that IS NOT a reset/reload error
+   // then break out of the retry loop and return error to user.
+   // If no error (a successful call) then delete any previous error
+   // (don't want to confuse caller) and break out of retry loop
+   for (int i = 0; i <= HBRT_FW_REQUEST_RETRIES; ++i)
+   {
+      // Make the firmware_request call
+      int rc = g_hostInterfaces->firmware_request(i_reqLen,  i_req,
+                                                  o_respLen, o_resp);
+
+      // Not all request messages that fail require an error log, most do
+      bool l_createErrorLog = true;
+
+      // The error log user data 1 and user data 2 will change
+      // based on response message type
+      uint64_t l_userData1(0), l_userData2(0);
+
+      // Error with FSP resetting and/or reloading? If so, then retry
+      if (HBRT_RC_FSPDEAD == rc)
+      {
+         // This is a reset/reload err
+         l_isResetReloadErr = true;
+
+         // Commit any previous errors
+         if (l_err)
+         {
+            // Commit any previous error log
+            errlCommit(l_err, RUNTIME_COMP_ID);
+         }
+
+         // Print trace info based on request message and populate error
+         // log user data 1 and user data 2 based on request message
+         switch (l_req_fw_msg->io_type)
+         {
+            case hostInterfaces::HBRT_FW_MSG_TYPE_REQ_HCODE_UPDATE:
+                 {
+                    TRACFCOMP(g_trac_runtime,
+                           ERR_MRK"FSP is doing a reset/reload, "
+                           "HCODE update failed. "
+                           "retry:%d/%d, rc:%d, io_type:%d, chipId:0x%llX, "
+                           "section:0x%X, operation:0x%X, scomAddr:0x%llX, "
+                           "scomData:0x%llX",
+                           i,
+                           HBRT_FW_REQUEST_RETRIES,
+                           rc,
+                           l_req_fw_msg->io_type,
+                           l_req_fw_msg->req_hcode_update.i_chipId,
+                           l_req_fw_msg->req_hcode_update.i_section,
+                           l_req_fw_msg->req_hcode_update.i_operation,
+                           l_req_fw_msg->req_hcode_update.i_scomAddr,
+                           l_req_fw_msg->req_hcode_update.i_scomData);
+
+                    // Pack user data 1 with Hypervisor return code and
+                    // firmware request message type
+                    l_userData1 = TWO_UINT32_TO_UINT64(rc,
+                                                       l_req_fw_msg->io_type);
+
+                    // Pack user data 2 with HCODE scom address and
+                    // HCODE scom data
+                    l_userData2 = TWO_UINT32_TO_UINT64(
+                                  l_req_fw_msg->req_hcode_update.i_scomAddr,
+                                  l_req_fw_msg->req_hcode_update.i_scomData);
+                 }
+            break;//END case hostInterfaces::HBRT_FW_MSG_TYPE_REQ_HCODE_UPDATE:
+
+            case hostInterfaces::HBRT_FW_MSG_TYPE_ERROR_LOG:
+                 {
+                    TRACFCOMP(g_trac_runtime,
+                              ERR_MRK"FSP is doing a reset/reload, "
+                                     "sending error log to FSP failed. "
+                                     "retry:%d/%d, rc:%d, plid:0x%08x",
+                              i,
+                              HBRT_FW_REQUEST_RETRIES,
+                              rc,
+                              l_req_fw_msg->error_log.i_plid);
+
+                    // Don't create an error log for this request type
+                    l_createErrorLog = false;
+                 }
+            break; // END case hostInterfaces::HBRT_FW_MSG_TYPE_ERROR_LOG:
+
+            case hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ:
+                 {
+                    TRACFCOMP(g_trac_runtime,
+                             "FSP is doing a reset/reload, "
+                             "doing a VPD write failed. "
+                             "retry:%d/%d, rc:%d, io_type:%d, dataSize:%d, "
+                             "seqnum:0x%X, msgq:0x%X, msgType:0x%X, __req:%d, "
+                             "__onlyError:%d",
+                             i,
+                             HBRT_FW_REQUEST_RETRIES,
+                             rc,
+                             l_req_fw_msg->io_type,
+                             l_req_fw_msg->generic_msg.dataSize,
+                             l_req_fw_msg->generic_msg.seqnum,
+                             l_req_fw_msg->generic_msg.msgq,
+                             l_req_fw_msg->generic_msg.msgType,
+                             l_req_fw_msg->generic_msg.__req,
+                             l_req_fw_msg->generic_msg.__onlyError);
+
+                    // Pack user data 1 with Hypervisor return code and
+                    // firmware request message sequence number
+                    l_userData1 = TWO_UINT32_TO_UINT64(
+                                           rc,
+                                           l_req_fw_msg->generic_msg.seqnum);
+
+                    // Pack user data 2 with message queue and message type
+                    l_userData2 = TWO_UINT32_TO_UINT64(
+                                            l_req_fw_msg->generic_msg.msgq,
+                                            l_req_fw_msg->generic_msg.msgType);
+                 }
+            break; // END case hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ:
+
+            default:
+            break;
+         }  // END switch (l_req_fw_msg->io_type)
+
+         if (l_createErrorLog)
+         {
+            /*@
+             * @errortype
+             * @severity         ERRL_SEV_INFORMATIONAL
+             * @moduleid         MOD_RT_FIRMWARE_REQUEST
+             * @reasoncode       RC_FW_REQUEST_RESET_RELOAD_ERR
+             * @userdata1[0:31]  Hypervisor return code
+             * @userdata1[32:63] Firmware Request type (HCODE Update) ||
+                                 sequence number (FSP MSG)
+             * @userdata2[0:31]  SCOM address (HCODE Update) ||
+                                 MBOX message type (FSP MSG)
+             * @userdata2[32:63] SCOM data (HCODE Update) ||
+                                 Message Type (FSP MSG)
+             * @devdesc          The Firmware Request call failed
+             */
+            l_err = new ErrlEntry(ERRL_SEV_INFORMATIONAL,
+                                  MOD_RT_FIRMWARE_REQUEST,
+                                  RC_FW_REQUEST_RESET_RELOAD_ERR,
+                                  l_userData1,
+                                  l_userData2,
+                                  true);
+         } // END if (l_createErrorLog)
+
+         // Try the firmware_request call again
+         continue;
+      }  // END if (hbrt_rc_fspdead == rc)
+      // Error is not with FSP resetting and/or reloading, so log the error
+      else if (rc)
+      {
+         if (l_err)
+         {
+            // Commit any previous reset/reload error log
+            errlCommit(l_err, RUNTIME_COMP_ID);
+         }
+
+         // This NOT a reset/reload error
+         l_isResetReloadErr = false;
+
+         // Print trace info based on request message type and populate
+         // error log user data 1 & user data 2 based on response message
+         switch (l_req_fw_msg->io_type)
+         {
+            case hostInterfaces::HBRT_FW_MSG_TYPE_REQ_HCODE_UPDATE:
+                 {
+                    // Default user data 1 wirh Hypervisor return code
+                    // and firmware request message type
+                    l_userData1 = TWO_UINT32_TO_UINT64(
+                                               rc,
+                                               l_req_fw_msg->io_type);
+
+                    // Pack user data 1 with Hypervisor return code and
+                    // generic response code if the response is of type
+                    // "RESP_GENERIC" else leave as is
+                    // 1st check if the response size is large enough
+                    //      for the type needed
+                    // 2nd make sure the message type is of
+                    //     type HBRT_FW_MSG_TYPE_RESP_GENERIC
+                    // 3rd make sure there is actaully a value in the status
+                    // If all these conditions are correct,
+                    //     then extract HWSV error
+                    if ((*o_respLen >=
+                                    (hostInterfaces::HBRT_FW_MSG_BASE_SIZE +
+                                    sizeof(l_resp_fw_msg->resp_generic)))  &&
+                         hostInterfaces::HBRT_FW_MSG_TYPE_RESP_GENERIC ==
+                                    l_resp_fw_msg->io_type &&
+                         0 != l_resp_fw_msg->resp_generic.o_status )
+                    {
+                       l_userData1 = TWO_UINT32_TO_UINT64(
+                                        rc,
+                                        l_resp_fw_msg->resp_generic.o_status);
+                    } // END if (*o_respLen >= ...
+
+                    // Pack user data 2 with HCODE scom
+                    // and HCODE scom data
+                    l_userData2 = TWO_UINT32_TO_UINT64(
+                                  l_req_fw_msg->req_hcode_update.i_scomAddr,
+                                  l_req_fw_msg->req_hcode_update.i_scomData);
+
+                    TRACFCOMP(g_trac_runtime,
+                           ERR_MRK"Failed doing an HCODE update. "
+                           "rc:%d, io_type:%d, chipId:0x%llX, section:0x%X, "
+                           "operation:0x%X, scomAddr:0x%llX, scomData:0x%llX",
+                           rc,
+                           l_req_fw_msg->io_type,
+                           l_req_fw_msg->req_hcode_update.i_chipId,
+                           l_req_fw_msg->req_hcode_update.i_section,
+                           l_req_fw_msg->req_hcode_update.i_operation,
+                           l_req_fw_msg->req_hcode_update.i_scomAddr,
+                           l_req_fw_msg->req_hcode_update.i_scomData);
+
+                 }
+            break;//END case hostInterfaces::HBRT_FW_MSG_TYPE_REQ_HCODE_UPDATE:
+
+            case hostInterfaces::HBRT_FW_MSG_TYPE_ERROR_LOG:
+                 {
+                    TRACFCOMP(g_trac_runtime,
+                              ERR_MRK"Failed sending error log to FSP "
+                              "via firmware_request. rc:%d, plid:0x%08X",
+                              rc,
+                              l_req_fw_msg->error_log.i_plid);
+
+                    // Don't create an error log for this request type
+                    l_createErrorLog = false;
+                 }
+            break; // END case hostInterfaces::HBRT_FW_MSG_TYPE_ERROR_LOG:
+
+
+            case hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ:
+                 {
+                    TRACFCOMP(g_trac_runtime,
+                             ERR_MRK"Failed doing a VPD write. "
+                             "rc:%d, io_type:%d, dataSize:%d, "
+                             "seqnum:0x%X, msgq:0x%X, msgType:0x%X, __req:%d, "
+                             "__onlyError:%d",
+                             rc,
+                             l_req_fw_msg->io_type,
+                             l_req_fw_msg->generic_msg.dataSize,
+                             l_req_fw_msg->generic_msg.seqnum,
+                             l_req_fw_msg->generic_msg.msgq,
+                             l_req_fw_msg->generic_msg.msgType,
+                             l_req_fw_msg->generic_msg.__req,
+                             l_req_fw_msg->generic_msg.__onlyError);
+
+                    // Pack user data 1 with Hypervisor return code and
+                    // firmware request message sequence number
+                    l_userData1 = TWO_UINT32_TO_UINT64(
+                                            rc,
+                                            l_req_fw_msg->generic_msg.seqnum);
+
+                    // Pack user data 2 with message queue and message type
+                    l_userData2 = TWO_UINT32_TO_UINT64(
+                                            l_req_fw_msg->generic_msg.msgq,
+                                            l_req_fw_msg->generic_msg.msgType);
+                 }
+            break; // END case hostInterfaces::HBRT_FW_MSG_HBRT_FSP_REQ:
+
+            default:
+               break;
+         }  // END switch (l_req_fw_msg->io_type)
+
+         if (l_createErrorLog)
+         {
+            /*@
+             * @errortype
+             * @severity         ERRL_SEV_PREDICTIVE
+             * @moduleid         MOD_RT_FIRMWARE_REQUEST
+             * @reasoncode       RC_FW_REQUEST_ERR
+             * @userdata1[0:31]  Hypervisor return code
+             * @userdata1[32:63] Firmware Request type (HCODE Update) ||
+                                 sequence number (FSP MSG)
+             * @userdata2[0:31]  SCOM address (HCODE Update) ||
+                                 MBOX message type (FSP MSG)
+             * @userdata2[32:63] SCOM data (HCODE Update) ||
+                                 Message Type (FSP MSG)
+             * @devdesc          The Firmware Request call failed
+             */
+            l_err = new ErrlEntry(ERRL_SEV_PREDICTIVE,
+                                  MOD_RT_FIRMWARE_REQUEST,
+                                  RC_FW_REQUEST_ERR,
+                                  l_userData1,
+                                  l_userData2,
+                                  true);
+          } // END if (l_createErrorLog)
+
+         // Break out of the retry loop
+         break;
+      } // END else if (rc) generic_msg
+      // There was no RC returned from the FSP request, there could be an issue
+      // with HWSV.  If so then a PLID will be returned in the first
+      // 4 bytes of the 'data' member for an HBRT_FW_MSG_HBRT_FSP_RESP type.
+      // 1st check if the response size is large enough for the type needed
+      // 2nd make sure the message is of HBRT_FW_MSG_HBRT_FSP_RESP type.
+      // 3rd check if the 'data' member will contain an error ID (PLID)
+      // 4th make sure the PLID has a value
+      // If all these conditions are correct, then pull out PLID
+      else if ((*o_respLen >=
+               (hostInterfaces::HBRT_FW_MSG_BASE_SIZE +
+                      sizeof(hostInterfaces::hbrt_fw_msg::generic_msg))) &&
+               (hostInterfaces::HBRT_FW_MSG_HBRT_FSP_RESP ==
+                      l_resp_fw_msg->io_type) &&
+               (GenericFspMboxMessage_t::ERROR_ONLY ==
+                      l_resp_fw_msg->generic_msg.__onlyError) &&
+               (0 != (l_resp_fw_msg->generic_msg.data >> 32)))
+
+      {
+         if (l_err)
+         {
+            // Commit any previous reset/reload error log
+            errlCommit(l_err, RUNTIME_COMP_ID);
+         }
+
+         // This NOT a reset/reload error
+         l_isResetReloadErr = false;
+
+         uint32_t l_errPlid = l_resp_fw_msg->generic_msg.data >> 32;
+
+         TRACFCOMP(g_trac_runtime,
+                   ERR_MRK"Failed doing a VPD write. "
+                          "rc:%d, plid:0x%X, io_type:%d, dataSize:%d, "
+                          "seqnum:0x%X, msgq:0x%X, msgType:0x%X, __req:%d, "
+                          "__onlyError:%d",
+                          rc,
+                          l_errPlid,
+                          l_req_fw_msg->io_type,
+                          l_req_fw_msg->generic_msg.dataSize,
+                          l_req_fw_msg->generic_msg.seqnum,
+                          l_req_fw_msg->generic_msg.msgq,
+                          l_req_fw_msg->generic_msg.msgType,
+                          l_req_fw_msg->generic_msg.__req,
+                          l_req_fw_msg->generic_msg.__onlyError);
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_PREDICTIVE
+          * @moduleid         MOD_RT_FIRMWARE_REQUEST
+          * @reasoncode       RC_FW_REQUEST_HWSV_ERR
+          * @userdata1[0:31]  HWSV error log id (PLID)
+          * @userdata1[32:63] Sequence number
+          * @userdata2[0:31]  MBOX message type
+          * @userdata2[32:63] Message Type
+          * @devdesc          The Firmware Request call failed
+          */
+         l_err = new ErrlEntry(ERRL_SEV_PREDICTIVE,
+                               MOD_RT_FIRMWARE_REQUEST,
+                               RC_FW_REQUEST_HWSV_ERR,
+                               TWO_UINT32_TO_UINT64(
+                                            l_errPlid,
+                                            l_req_fw_msg->generic_msg.seqnum),
+                               TWO_UINT32_TO_UINT64(
+                                            l_req_fw_msg->generic_msg.msgq,
+                                            l_req_fw_msg->generic_msg.msgType),
+                               true);
+
+         // Break out of the retry loop
+         break;
+      }
+      else
+      {
+         // Break out of the retry loop, the call was successful
+         // But first delete any previous errors.  It can be confusing
+         // if an error is committed or returned on a successful call
+         delete l_err, l_err = nullptr;
+         break;
+      }
+   } // END for (int i = 0; i <= HBRT_FW_REQUEST_RETRIES; ++i)
+
+   // If there is an error, then call addFFDC
+   // with request/response sructs
+   if (l_err)
+   {
+      // Hit the max attempts (currently should only one retry) and hitting
+      // reset/reload error
+      if (l_isResetReloadErr)
+      {
+         // Change the severity of a reset/reload error to predictive
+         l_err->setSev(ERRL_SEV_PREDICTIVE);
+      }
+
+      l_err->addFFDC(RUNTIME_COMP_ID,
+                     l_req_fw_msg,
+                     i_reqLen,
+                     0, 0, false );
+
+      if (*o_respLen > 0)
+      {
+         l_err->addFFDC(RUNTIME_COMP_ID,
+                        l_resp_fw_msg,
+                        *o_respLen,
+                        0, 0, false );
+      }
+
+      l_err->collectTrace( "FW_REQ", 256);
+   }  // END if (l_err)
+
+  return l_err;
+};