Centaur Channel Checkstop (runtime)

This story covers when a DMI channel gets checkstopped at runtime
and how to service the subsequent SCOM operations that PRD
requests to diagnose the issue.  At runtime, PRD makes all of
its SCOM requests to HBRT which passes them on to PHYP for
in-band access.  The checkstop blocks in-band access, so HBRT
must fail-over to messaging the FSP to do FSI accesses.

To reduce the number of SCOM calls to the FSP, HBRT makes
a "multi-SCOM read" call for all the common SCOMs that
PRD will request after a checkstop, and caches the results.

Two new calls (MBOX, generic messaging interface) to the FSP
were added in a previous commit to allow FSI SCOM operations
through the FSP:
- MSG_SINGLE_SCOM_OP
- MSG_MULTI_SCOM_OP

Also, a new map, chnlFailScomList, was added in a previous
commit.  chnlFailScomList contains a list of target types
and associated SCOM addresses that PRD is likely want to
read after a checkstop.  PRD is responsible for maintaining
the contents.

Change-Id: I829a72067007ac8a61d80caa690d8eedee0f08cc
RTC:189294
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59197
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

12 files changed, 938 insertions, 25 deletions

diff --git a/src/include/runtime/generic_hbrt_fsp_message.H b/src/include/runtime/generic_hbrt_fsp_message.H
index 6daff3cc2..30566290d 100644
--- a/src/include/runtime/generic_hbrt_fsp_message.H
+++ b/src/include/runtime/generic_hbrt_fsp_message.H
@@ -73,7 +73,7 @@ struct TargetDeconfigHbrtFspData_t
   */
 struct SingleScomOpHbrtFspData_t
 {
-    uint8_t     scom_op;    // 0 == read, 1 == write
+    uint8_t     scom_op;    // DeviceFW::READ, DeviceFW::WRITE
     uint32_t    huid;       // hardware target
     uint64_t    scom_addr;
     uint64_t    scom_data;  // SCOM value read by FSP, or
@@ -92,11 +92,12 @@ struct MultiScomReadHbrtFspData_t
     uint8_t     scom_num;   // number of SCOMs to read
     uint64_t    scom_data;  // addresses of SCOMs to read, or
                             // values of SCOMs in FSP response
+                            //
                             // ** if a SCOM cannot be read, its value **
                             // ** should be returned as DEADBEEF      **
                             //
                             // placeholder that can be cast to an array
-                            // of values (or addresses)
+                            // of SCOM addresses (or values)
                             // uint64_t* mydata =
                             //              (uint64_t*)&(l_generic_msg.data);
                             // more than 8 bytes of data is possible
diff --git a/src/include/usr/fsiscom/fsiscom_reasoncodes.H b/src/include/usr/fsiscom/fsiscom_reasoncodes.H
index 0c9aa0767..2acee4c72 100644
--- a/src/include/usr/fsiscom/fsiscom_reasoncodes.H
+++ b/src/include/usr/fsiscom/fsiscom_reasoncodes.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -29,19 +31,22 @@ namespace FSISCOM
 {
     enum FSISCOMModuleId
     {
-        MOD_FSISCOM_INVALID           = 0x00, /**< Zero is an invalid module id */
-        MOD_FSISCOM_PERFORMOP         = 0x01, /**< fsiscom.C : fsiScomPerformOp */
-
+        MOD_FSISCOM_INVALID             = 0x00, // Zero is an invalid module id
+        MOD_FSISCOM_PERFORM_OP          = 0x01, // fsiscom.C : fsiScomPerformOp
+        MOD_FSISCOM_RT_SEND_SCOM_TO_FSP = 0x02,
+        MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP = 0x03,
     };
 
     enum FSISCOMReasonCode
     {
-        RC_INVALID                   = FSISCOM_COMP_ID | 0x00, 
-        RC_WRITE_ERROR               = FSISCOM_COMP_ID | 0x01, 
+        RC_INVALID                   = FSISCOM_COMP_ID | 0x00,
+        RC_WRITE_ERROR               = FSISCOM_COMP_ID | 0x01,
         RC_READ_ERROR                = FSISCOM_COMP_ID | 0x02,
         RC_INVALID_LENGTH            = FSISCOM_COMP_ID | 0x03,
         RC_INVALID_ADDRESS           = FSISCOM_COMP_ID | 0x04,
         RC_INVALID_OPTYPE            = FSISCOM_COMP_ID | 0x05,
+        RC_RT_INTERFACE_ERR          = FSISCOM_COMP_ID | 0x06,
+        RC_RT_NULL_FW_MSG_PTR        = FSISCOM_COMP_ID | 0x07,
     };
 };
 
diff --git a/src/include/usr/scom/runtime/rt_scomif.H b/src/include/usr/scom/runtime/rt_scomif.H
index 4039f2827..5c645bde0 100644
--- a/src/include/usr/scom/runtime/rt_scomif.H
+++ b/src/include/usr/scom/runtime/rt_scomif.H
@@ -37,7 +37,7 @@ namespace SCOM
  * @param[in]           i_target         Scom target
  * @param[in]           i_scomAddr       Scom address
  * @param[in|out]       io_buffer        Pointer to scom data
- * @return              errlHndl_t
+ * @return  errlHndl_t
  */
 errlHndl_t sendScomToHyp(DeviceFW::OperationType i_opType,
                          TARGETING::Target * i_target,
@@ -46,4 +46,44 @@ errlHndl_t sendScomToHyp(DeviceFW::OperationType i_opType,
 
 };  // end namespace SCOM
 
+namespace FSISCOM
+{
+
+/**
+ * @brief DMI channel has checkstopped.  Mark it bad and switch to FSP access.
+ *
+ * @param[in]     i_target    SCOM target
+ * @return  None
+ */
+void switchToFspScomAccess(TARGETING::TargetHandle_t i_target);
+
+/**
+ * @brief This function sends the scom op to the FSP
+ *
+ * @param[in]     i_opType    Scom operation type, see driverif.H
+ * @param[in]     i_target    Scom target
+ * @param[in]     i_scomAddr  Scom address
+ * @param[in/out] io_buffer   Read: Pointer to output data storage
+ *                            Write: Pointer to input data storage
+ * @return  errlHndl_t
+ */
+errlHndl_t sendScomOpToFsp(DeviceFW::OperationType   i_opType,
+                           TARGETING::TargetHandle_t i_target,
+                           uint64_t                  i_scomAddr,
+                           void *                    io_buffer);
+
+/**
+ * @brief Ask FSP to read list of SCOMs
+ *
+ * @param[in]     i_target    Scom target
+ * @param[in]     i_scomAddr  Scom addresses to read
+ * @param[out]    o_scomValue Scom values read (0xDEADBEEF for errors)
+ * @return  errlHndl_t
+ */
+errlHndl_t sendMultiScomReadToFsp(TARGETING::TargetHandle_t i_target,
+                                  std::vector<uint64_t> &   i_scomAddr,
+                                  std::vector<uint64_t> &   o_scomValue);
+
+};  // end namespace FSISCOM
+
 #endif  // end __RT_SCOMIF_H
diff --git a/src/makefile b/src/makefile
index f2b570dcc..5a278a265 100644
--- a/src/makefile
+++ b/src/makefile
@@ -278,6 +278,7 @@ RUNTIME_MODULES += targeting_rt
 RUNTIME_MODULES += util_rt
 RUNTIME_MODULES += devicefw_rt
 RUNTIME_MODULES += xscom_rt
+RUNTIME_MODULES += fsiscom_rt
 RUNTIME_MODULES += ibscom_rt
 RUNTIME_MODULES += scom_rt
 RUNTIME_MODULES += vpd_rt
diff --git a/src/usr/fsiscom/fsiscom.C b/src/usr/fsiscom/fsiscom.C
index 9a35c9b62..dd2b26271 100644
--- a/src/usr/fsiscom/fsiscom.C
+++ b/src/usr/fsiscom/fsiscom.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -200,7 +200,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
             TRACFCOMP( g_trac_fsiscom, ERR_MRK "fsiScomPerformOp> Invalid data length : io_buflen=%d", io_buflen );
             /*@
              * @errortype
-             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORMOP
+             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORM_OP
              * @reasoncode   FSISCOM::RC_INVALID_LENGTH
              * @userdata1    SCOM Address
              * @userdata2    Data Length
@@ -209,7 +209,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
              *               Invalid data length for a SCOM operation.
              */
             l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                            FSISCOM::MOD_FSISCOM_PERFORMOP,
+                                            FSISCOM::MOD_FSISCOM_PERFORM_OP,
                                             FSISCOM::RC_INVALID_LENGTH,
                                             l_scomAddr,
                                             TO_UINT64(io_buflen));
@@ -225,7 +225,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
             TRACFCOMP( g_trac_fsiscom, ERR_MRK "fsiScomPerformOp> Address contains more than 31 bits : l_scomAddr=0x%.16X", l_scomAddr );
             /*@
              * @errortype
-             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORMOP
+             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORM_OP
              * @reasoncode   FSISCOM::RC_INVALID_ADDRESS
              * @userdata1    SCOM Address
              * @userdata2    Target HUID
@@ -235,7 +235,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
              *               Invalid address on a SCOM operation.
              */
             l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                            FSISCOM::MOD_FSISCOM_PERFORMOP,
+                                            FSISCOM::MOD_FSISCOM_PERFORM_OP,
                                             FSISCOM::RC_INVALID_ADDRESS,
                                             l_scomAddr,
                                             TARGETING::get_huid(i_target));
@@ -322,7 +322,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
                 TRACFCOMP( g_trac_fsiscom, ERR_MRK"fsiScomPerformOp:Write: PCB/PIB error received: l_status=0x%X)", l_status);
                 /*@
                  * @errortype
-                 * @moduleid     FSISCOM::MOD_FSISCOM_PERFORMOP
+                 * @moduleid     FSISCOM::MOD_FSISCOM_PERFORM_OP
                  * @reasoncode   FSISCOM::RC_WRITE_ERROR
                  * @userdata1    SCOM Addr
                  * @userdata2[00:31]  Target HUID
@@ -334,7 +334,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
                  */
                 l_err = new ERRORLOG::ErrlEntry(
                                 ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                FSISCOM::MOD_FSISCOM_PERFORMOP,
+                                FSISCOM::MOD_FSISCOM_PERFORM_OP,
                                 FSISCOM::RC_WRITE_ERROR,
                                 l_scomAddr,
                                 TWO_UINT32_TO_UINT64(
@@ -389,7 +389,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
 
                 /*@
                  * @errortype
-                 * @moduleid     FSISCOM::MOD_FSISCOM_PERFORMOP
+                 * @moduleid     FSISCOM::MOD_FSISCOM_PERFORM_OP
                  * @reasoncode   FSISCOM::RC_READ_ERROR
                  * @userdata1    SCOM Addr
                  * @userdata2[00:31]  Target HUID
@@ -399,7 +399,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
                  *               Error returned from SCOM engine after read.
                  */
                 l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                                FSISCOM::MOD_FSISCOM_PERFORMOP,
+                                                FSISCOM::MOD_FSISCOM_PERFORM_OP,
                                                 FSISCOM::RC_READ_ERROR,
                                                 l_scomAddr,
                                                 TWO_UINT32_TO_UINT64(
@@ -448,7 +448,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
 
             /*@
              * @errortype
-             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORMOP
+             * @moduleid     FSISCOM::MOD_FSISCOM_PERFORM_OP
              * @reasoncode   FSISCOM::RC_INVALID_OPTYPE
              * @userdata1[0:31]    Operation Type (i_opType) : 0=READ, 1=WRITE
              * @userdata1[32:64]   Input scom address
@@ -458,7 +458,7 @@ errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
              *               Unsupported SCOM operation type.
              */
             l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
-                                            FSISCOM::MOD_FSISCOM_PERFORMOP,
+                                            FSISCOM::MOD_FSISCOM_PERFORM_OP,
                                             FSISCOM::RC_INVALID_OPTYPE,
                                             TWO_UINT32_TO_UINT64(i_opType,
                                                                  l_scomAddr),
diff --git a/src/usr/fsiscom/makefile b/src/usr/fsiscom/makefile
index 19df46488..ba897a677 100644
--- a/src/usr/fsiscom/makefile
+++ b/src/usr/fsiscom/makefile
@@ -5,7 +5,9 @@
 #
 # OpenPOWER HostBoot Project
 #
-# COPYRIGHT International Business Machines Corp. 2011,2014
+# Contributors Listed Below - COPYRIGHT 2011,2018
+# [+] International Business Machines Corp.
+#
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,5 +28,6 @@ MODULE = fsiscom
 OBJS += fsiscom.o
 
 SUBDIRS += test.d
+SUBDIRS += runtime.d
 
 include ${ROOTPATH}/config.mk
diff --git a/src/usr/fsiscom/runtime/makefile b/src/usr/fsiscom/runtime/makefile
new file mode 100644
index 000000000..48d1ebcb5
--- /dev/null
+++ b/src/usr/fsiscom/runtime/makefile
@@ -0,0 +1,33 @@
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/usr/fsiscom/runtime/makefile $
+#
+# OpenPOWER HostBoot Project
+#
+# Contributors Listed Below - COPYRIGHT 2018
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# IBM_PROLOG_END_TAG
+HOSTBOOT_RUNTIME = 1
+ROOTPATH = ../../../..
+MODULE = fsiscom_rt
+
+OBJS += rt_fsiscom.o
+
+EXTRAINCDIR += ${ROOTPATH}/src/usr/diag/prdf/
+VPATH += ..
+include $(ROOTPATH)/config.mk
diff --git a/src/usr/fsiscom/runtime/rt_fsiscom.C b/src/usr/fsiscom/runtime/rt_fsiscom.C
new file mode 100644
index 000000000..69c4ccd6b
--- /dev/null
+++ b/src/usr/fsiscom/runtime/rt_fsiscom.C
@@ -0,0 +1,595 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/fsiscom/runtime/rt_fsiscom.C $                        */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2018                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#include <devicefw/driverif.H>
+#include <trace/interface.H>
+#include <errl/errlentry.H>
+#include <errl/errlmanager.H>
+#include <fsiscom/fsiscom_reasoncodes.H>
+#include "../fsiscom.H"
+#include <scom/scomif.H>
+#include <scom/runtime/rt_scomif.H>
+#include <targeting/common/utilFilter.H>
+
+#include <runtime/hbrt_utilities.H>        // createGenericFspMsg
+#include <util/runtime/rt_fwreq_helper.H>  // firmware_request_helper
+#include <plat/cen/prdfCenChnlFailCache.H> // chnlFailScomList
+
+#include <map>
+
+// Trace definition
+trace_desc_t* g_trac_fsiscom = NULL;
+TRAC_INIT(&g_trac_fsiscom, FSISCOM_COMP_NAME, 2*KILOBYTE, TRACE::BUFFER_SLOW);
+
+namespace FSISCOM
+{
+
+/**
+ * @brief This function sends the scom op to the FSP
+ *
+ * @param[in]     i_opType    Scom operation type, see driverif.H
+ * @param[in]     i_target    Scom target
+ * @param[in]     i_scomAddr  Scom address
+ * @param[in/out] io_buffer   Read: Pointer to output data storage
+ *                            Write: Pointer to input data storage
+ * @return  errlHndl_t
+ */
+errlHndl_t sendScomOpToFsp(DeviceFW::OperationType i_opType,
+                           TARGETING::TargetHandle_t i_target,
+                           uint64_t i_scomAddr,
+                           void * io_buffer)
+{
+   errlHndl_t l_err = nullptr;
+
+   // Handles to the firmware messages
+   hostInterfaces::hbrt_fw_msg *l_req_fw_msg  = nullptr;
+   hostInterfaces::hbrt_fw_msg *l_resp_fw_msg = nullptr;
+
+   do
+   {
+      if ((nullptr == g_hostInterfaces) ||
+          (nullptr == g_hostInterfaces->firmware_request))
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "Hypervisor firmware_request interface not linked");
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_UNRECOVERABLE
+          * @moduleid         MOD_FSISCOM_RT_SEND_SCOM_TO_FSP
+          * @reasoncode       RC_RT_INTERFACE_ERR
+          * @userdata1        target's HUID
+          * @userdata2        SCOM address
+          * @devdesc          Hypervisor firmware request interface not linked
+          */
+         l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                         MOD_FSISCOM_RT_SEND_SCOM_TO_FSP,
+                                         RC_RT_INTERFACE_ERR,
+                                         get_huid(i_target),
+                                         i_scomAddr,
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+         break;
+      }
+
+      // Create and initialize to zero a few needed variables
+      uint32_t l_fsp_data_size(0);
+      uint64_t l_req_fw_msg_size(0), l_resp_fw_msg_size(0);
+
+      // Create the dynamic firmware messages
+      createGenericFspMsg(sizeof(SingleScomOpHbrtFspData_t),
+                          l_fsp_data_size,
+                          l_req_fw_msg_size,
+                          l_req_fw_msg,
+                          l_resp_fw_msg_size,
+                          l_resp_fw_msg);
+
+      // If there was an issue with creating the messages,
+      // create an Error Log entry and exit
+      if (!l_req_fw_msg || !l_resp_fw_msg)
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "Unable to allocate firmware request messages");
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_UNRECOVERABLE
+          * @moduleid         MOD_FSISCOM_RT_SEND_SCOM_TO_FSP
+          * @reasoncode       RC_RT_NULL_FW_MSG_PTR
+          * @userdata1        target's HUID
+          * @userdata2        SCOM address
+          * @devdesc          Unable to allocate firmware request messages
+          */
+         l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                         MOD_FSISCOM_RT_SEND_SCOM_TO_FSP,
+                                         RC_RT_NULL_FW_MSG_PTR,
+                                         get_huid(i_target),
+                                         i_scomAddr,
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+         break;
+      }
+
+      // Populate the request message with given data
+      l_req_fw_msg->generic_msg.msgq = MBOX::FSP_SCOM_OPS_MSGQ;
+      l_req_fw_msg->generic_msg.msgType =
+                                GenericFspMboxMessage_t::MSG_SINGLE_SCOM_OP;
+
+      // Create a useful struct to populate the generic_msg::data field
+      // Set the command-specific portion of the FSP message
+      SingleScomOpHbrtFspData_t* l_fspData =
+                  reinterpret_cast<SingleScomOpHbrtFspData_t*>
+                                 (&(l_req_fw_msg->generic_msg.data));
+      l_fspData->scom_op   = i_opType;
+      l_fspData->huid      = get_huid(i_target);
+      l_fspData->scom_addr = i_scomAddr;
+      l_fspData->scom_data = *((uint64_t *)io_buffer);
+
+      // Make the firmware_request call
+      // Ask the FSP to perform this SCOM operation
+      TRACFCOMP(g_trac_fsiscom, "Sending SINGLE_SCOM_OP firmware_request, "
+                                "op=%c, huid=0x%X, addr=%llX",
+                                (i_opType == DeviceFW::READ) ? 'r' : 'w',
+                                l_fspData->huid, l_fspData->scom_addr);
+      l_err = firmware_request_helper(l_req_fw_msg_size,
+                                      l_req_fw_msg,
+                                      &l_resp_fw_msg_size,
+                                      l_resp_fw_msg);
+
+      if (l_err)
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "FSP scom read/write failed. "
+                   "target 0x%llX addr 0x%llX r/w %d",
+                   get_huid(i_target), i_scomAddr, i_opType);
+
+         break;
+      }
+
+      // Create a useful struct to populate the generic_msg::data field
+      // Get the command-specific portion of the returned FSP message
+      l_fspData =
+                  reinterpret_cast<SingleScomOpHbrtFspData_t*>
+                                 (&(l_resp_fw_msg->generic_msg.data));
+      *((uint64_t *)io_buffer) = l_fspData->scom_data;
+   } while(0);
+
+   // Release the firmware messages and set to NULL
+   delete [] l_req_fw_msg;
+   delete [] l_resp_fw_msg;
+   l_req_fw_msg = l_resp_fw_msg = nullptr;
+
+   return l_err;
+}
+
+
+/**
+ * @brief Ask FSP to read list of SCOMs
+ *
+ * @param[in]     i_target    Scom target
+ * @param[in]     i_scomAddr  Scom addresses to read
+ * @param[in/out] o_scomValue Scom values read (0xDEADBEEF for errors)
+ * @return  errlHndl_t
+ */
+errlHndl_t sendMultiScomReadToFsp(TARGETING::TargetHandle_t i_target,
+                                  std::vector<uint64_t> &i_scomAddr,
+                                  std::vector<uint64_t> &o_scomValue)
+{
+   errlHndl_t l_err = nullptr;
+
+   // Handles to the firmware messages
+   hostInterfaces::hbrt_fw_msg *l_req_fw_msg  = nullptr;
+   hostInterfaces::hbrt_fw_msg *l_resp_fw_msg = nullptr;
+
+   do
+   {
+      if ((nullptr == g_hostInterfaces) ||
+          (nullptr == g_hostInterfaces->firmware_request))
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "Hypervisor firmware_request interface not linked");
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_UNRECOVERABLE
+          * @moduleid         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP
+          * @reasoncode       RC_RT_INTERFACE_ERR
+          * @userdata1        target's HUID
+          * @userdata2        # of SCOMs to read
+          * @devdesc          Hypervisor firmware request interface not linked
+          */
+         l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP,
+                                         RC_RT_INTERFACE_ERR,
+                                         get_huid(i_target),
+                                         i_scomAddr.size(),
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+         break;
+      }
+
+      // Create and initialize to zero a few needed variables
+      uint32_t l_fsp_data_size(0);
+      uint64_t l_req_fw_msg_size(0), l_resp_fw_msg_size(0);
+
+      // Need to add the array of addresses/values
+      size_t   l_numScoms = i_scomAddr.size();
+      uint64_t l_msg_size = sizeof(MultiScomReadHbrtFspData_t);
+
+      // Default MultiScomRead message size is for only one SCOM, need to
+      // add any additional SCOMs to the message size in order to have the
+      // additional space at the end of the struct.
+      if (l_numScoms > 1)
+      {
+          l_msg_size += (l_numScoms - 1) * sizeof(uint64_t);
+      }
+
+      // Create the dynamic firmware messages
+      createGenericFspMsg(l_msg_size,
+                          l_fsp_data_size,
+                          l_req_fw_msg_size,
+                          l_req_fw_msg,
+                          l_resp_fw_msg_size,
+                          l_resp_fw_msg);
+
+      // If there was an issue with creating the messages,
+      // create an Error Log entry and exit
+      if (!l_req_fw_msg || !l_resp_fw_msg)
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "Unable to allocate firmware request messages");
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_UNRECOVERABLE
+          * @moduleid         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP
+          * @reasoncode       RC_RT_NULL_FW_MSG_PTR
+          * @userdata1        target's HUID
+          * @userdata2        # of SCOMs to read
+          * @devdesc          Unable to allocate firmware request messages
+          */
+         l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP,
+                                         RC_RT_NULL_FW_MSG_PTR,
+                                         get_huid(i_target),
+                                         i_scomAddr.size(),
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+         break;
+      }
+
+      // Populate the request message with given data
+      l_req_fw_msg->generic_msg.msgq = MBOX::FSP_SCOM_OPS_MSGQ;
+      l_req_fw_msg->generic_msg.msgType =
+                                GenericFspMboxMessage_t::MSG_MULTI_SCOM_OP;
+
+      // Create a useful struct to populate the generic_msg::data field
+      // Set the command-specific portion of the FSP message
+      MultiScomReadHbrtFspData_t* l_fspData =
+                  reinterpret_cast<MultiScomReadHbrtFspData_t*>
+                                 (&(l_req_fw_msg->generic_msg.data));
+      l_fspData->huid      = get_huid(i_target);
+      l_fspData->scom_num  = l_numScoms;
+      // copy SCOM addresses into scom_data
+      std::copy(i_scomAddr.begin(), i_scomAddr.end(), &(l_fspData->scom_data));
+
+      // Make the firmware_request call
+      // Ask the FSP to perform this SCOM operation
+      TRACFCOMP(g_trac_fsiscom, "Sending MULTI_SCOM_OP firmware_request");
+      l_err = firmware_request_helper(l_req_fw_msg_size,
+                                      l_req_fw_msg,
+                                      &l_resp_fw_msg_size,
+                                      l_resp_fw_msg);
+
+      if (l_err)
+      {
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "FSP multi-scom read failed. target 0x%llX",
+                   get_huid(i_target));
+
+         break;
+      }
+
+      // Create a useful struct to populate the generic_msg::data field
+      // Get the command-specific portion of the returned FSP message
+      l_fspData =
+                  reinterpret_cast<MultiScomReadHbrtFspData_t*>
+                                 (&(l_resp_fw_msg->generic_msg.data));
+      if (l_fspData->scom_num != l_numScoms)
+      {
+         // Can't continue because we don't know how the returned SCOM values
+         // match with the requested SCOM addresses.
+
+         TRACFCOMP(g_trac_fsiscom, ERR_MRK
+                   "FSP multi-scom read failed. target 0x%llX, "
+                   "SCOMs requested(%d) != SCOMs read(%d)",
+                   get_huid(i_target), l_numScoms, l_fspData->scom_num);
+
+         /*@
+          * @errortype
+          * @severity         ERRL_SEV_PREDICTIVE
+          * @moduleid         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP
+          * @reasoncode       RC_INVALID_LENGTH
+          * @userdata1        target's HUID
+          * @userdata2[00:31] # of SCOMs requested
+          * @userdata2[32:63] # of SCOMs returned
+          * @devdesc          Multi-SCOM read did not return correct SCOMs
+          */
+         l_err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_PREDICTIVE,
+                                         MOD_FSISCOM_RT_SEND_MULTI_SCOM_TO_FSP,
+                                         RC_INVALID_LENGTH,
+                                         get_huid(i_target),
+                                         TWO_UINT32_TO_UINT64(
+                                                         l_numScoms,
+                                                         l_fspData->scom_num),
+                                         ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+
+         break;
+      }
+      o_scomValue.insert(o_scomValue.end(),
+                         &(l_fspData->scom_data),
+                         &(l_fspData->scom_data) + l_numScoms);
+   } while(0);
+
+   // Release the firmware messages and set to NULL
+   delete [] l_req_fw_msg;
+   delete [] l_resp_fw_msg;
+   l_req_fw_msg = l_resp_fw_msg = nullptr;
+
+   return l_err;
+}
+
+// Mark the target and all the target's children as useFsiScom.
+static void markUseFsiScom(TARGETING::TargetHandle_t i_target)
+{
+    // Mark target
+    TARGETING::ScomSwitches l_switches = {0};
+    if (i_target->tryGetAttr<TARGETING::ATTR_SCOM_SWITCHES>(l_switches))
+    {
+        l_switches.useFsiScom = 1;
+        l_switches.useInbandScom = 0;
+        i_target->setAttr<TARGETING::ATTR_SCOM_SWITCHES>(l_switches);
+    }
+
+    // Mark children
+    TARGETING::TargetHandleList pChildList;
+    TARGETING::PredicateHwas    isFunctional;
+    isFunctional.functional(true);
+
+    TARGETING::targetService().getAssociated(pChildList, i_target,
+                                 TARGETING::TargetService::CHILD_BY_AFFINITY,
+                                 TARGETING::TargetService::ALL, &isFunctional);
+    for (auto& l_child: pChildList)
+    {
+        markUseFsiScom(l_child);
+    }
+}
+
+// This map is used to cache a bulk read of SCOM values that we expect PRD
+// to ask for.  A map of translated SCOM addresses and values is mapped to
+// the parent chip target.
+static std::map<TARGETING::Target *, std::map<uint64_t, uint64_t>> g_scomCache;
+
+/**
+ * @brief DMI channel has checkstopped.  Mark it bad and switch to FSP access.
+ *
+ * @param[in]     i_target    membuf target
+ * @return  None
+ */
+void switchToFspScomAccess(TARGETING::TargetHandle_t i_target)
+{
+    TRACDCOMP(g_trac_fsiscom,ENTER_MRK"switchToFspScomAccess");
+    errlHndl_t l_err = NULL;
+
+    // Need to mark the target's DMI channel and all targets below it
+    // as "useFsiScom" and bulk SCOM read requests should be sent to
+    // the FSP in order to populate the SCOM cache.
+    TARGETING::PredicateCTM     l_dmi(TARGETING::CLASS_UNIT,
+                                      TARGETING::TYPE_DMI);
+    TARGETING::TargetHandleList l_dmiList;
+
+    TARGETING::targetService().getAssociated(l_dmiList,
+                                  i_target,
+                                  TARGETING::TargetService::PARENT_BY_AFFINITY,
+                                  TARGETING::TargetService::ALL,
+                                  &l_dmi);
+    if (l_dmiList.size() != 1)
+    {
+        TRACFCOMP(g_trac_fsiscom,ERR_MRK"Unable to find DMI for Centaur.");
+    }
+    else
+    {
+        TARGETING::TargetHandle_t l_dmiParent = nullptr;
+
+        l_dmiParent = *(l_dmiList.begin());
+        markUseFsiScom(l_dmiParent);
+
+        // All of the cached SCOMs will map to the same membuf target (after
+        // SCOM address translation).
+        std::map<uint64_t, uint64_t> l_newmap;
+        TARGETING::TargetHandle_t l_membuf = nullptr;
+
+        if (i_target->getAttr<TARGETING::ATTR_TYPE>() ==
+                                                        TARGETING::TYPE_MEMBUF)
+        {
+            l_membuf = i_target;
+        }
+        else
+        {
+            l_membuf = const_cast<TARGETING::TargetHandle_t>
+                                          (TARGETING::getParentChip(i_target));
+        }
+
+        // Cache SCOMs that PRD will request.  chnlFailScomList is a map of
+        // target types (MEMBUF, MBA, etc) and the associated SCOMs for them.
+        // chnlFailScomList is maintained by PRD and defined in
+        // prdfCenChnlFailCache.H.
+        for (auto& l_typeScoms: PRDF::chnlFailScomList)
+        {
+            // For each target type in chnlFailScomList, find the associated
+            // targets and cache their SCOMs
+            TARGETING::TargetHandleList l_targetList;
+            getChildAffinityTargets(l_targetList, l_dmiParent,
+                                    TARGETING::CLASS_NA, l_typeScoms.first);
+
+            for (auto& l_target: l_targetList)
+            {
+                std::vector<uint64_t> l_scomVals;
+
+                l_scomVals.clear();
+                l_err = sendMultiScomReadToFsp(l_target,
+                                               l_typeScoms.second,
+                                               l_scomVals);
+                if (l_err)
+                {
+                    TRACFCOMP(g_trac_fsiscom,ERR_MRK
+                              "There was an error caching the SCOMs "
+                              "for huid(0x%llX)", get_huid(l_target));
+                    errlCommit(l_err, RUNTIME_COMP_ID);
+                    continue;
+                }
+
+                // Combine the requested SCOM addrs with the returned values in
+                // a local map, insert into cache keyed by target.  Don't save
+                // SCOMs with a returned value of 0xDEADBEEF.
+                for (size_t i = 0;i < l_typeScoms.second.size();++i)
+                {
+                    bool     l_needsWakeup = false;
+                    uint64_t l_relAddr     = 0; // relative SCOM address
+                    uint64_t l_transAddr   = 0; // translated SCOM address
+
+                    l_relAddr = l_typeScoms.second[i];
+                    l_transAddr = l_relAddr;
+
+                    if (l_target != l_membuf) // membuf addresses don't need
+                                              // translation
+                    {
+                        l_err = SCOM::scomTranslate(l_target,
+                                                    l_transAddr,
+                                                    l_needsWakeup);
+                    }
+
+                    if (l_err)
+                    {
+                        TRACFCOMP(g_trac_fsiscom,ERR_MRK
+                                  "There was an error translating the SCOM "
+                                  "address (0x%llX) for huid(0x%llX)",
+                                  l_relAddr, get_huid(l_target));
+                        errlCommit(l_err, RUNTIME_COMP_ID);
+                        continue;
+                    }
+
+                    if (l_scomVals[i] != 0xDEADBEEF)
+                    {
+                        l_newmap[l_transAddr] = l_scomVals[i];
+                    }
+                } // for SCOM address list
+            } // for l_targetList
+        } // for chnlFailScomList
+
+        // Copy local map into cache for later use.
+        g_scomCache[l_membuf] = l_newmap;
+    }
+
+    TRACDCOMP(g_trac_fsiscom,EXIT_MRK"switchToFspScomAccess");
+}
+
+
+// Direct Centaur FSI SCOM calls through this interface at runtime.
+// This is an alternate route for when a DMI channel checkstop has
+// occurred, and PHYP cannot service the operation.
+DEVICE_REGISTER_ROUTE(DeviceFW::WILDCARD,
+                      DeviceFW::FSISCOM,
+                      TARGETING::TYPE_MEMBUF,
+                      fsiScomPerformOp);
+
+/**
+ * @brief Complete the SCOM operation through the FSP.
+ *
+ * @param[in]     i_opType    Operation type, see driverif.H
+ * @param[in]     i_target    SCOM target
+ * @param[in/out] io_buffer   Read: Pointer to output data storage
+ *                            Write: Pointer to input data storage
+ * @param[in/out] io_buflen   Input: size of io_buffer (in bytes)
+ *                            Output: Read:  Size of output data
+ *                                    Write: Size of data written
+ * @param[in]   i_accessType  Access type
+ * @param[in]   i_args        This is an argument list for DD framework.
+ *                            In this function, there's only one argument,
+ *                            which is the SCOM address
+ * @return  errlHndl_t
+ */
+errlHndl_t fsiScomPerformOp(DeviceFW::OperationType i_opType,
+                            TARGETING::TargetHandle_t i_target,
+                            void* io_buffer,
+                            size_t& io_buflen,
+                            int64_t i_accessType,
+                            va_list i_args)
+{
+    TRACDCOMP(g_trac_fsiscom,ENTER_MRK"fsiScomPerformOp");
+    errlHndl_t l_err  = nullptr;
+    uint64_t   l_addr = va_arg(i_args,uint64_t);
+
+    l_err = SCOM::scomOpSanityCheck(i_opType,
+                                    i_target,
+                                    io_buffer,
+                                    io_buflen,
+                                    l_addr,
+                                    sizeof(uint64_t));
+
+    if (l_err)
+    {
+        // Trace here - sanity check does not know scom type
+        TRACFCOMP(g_trac_fsiscom,"Runtime FSIScom sanity check failed");
+    }
+    else
+    {
+        bool found = false;
+        auto targ = g_scomCache.find(i_target); // find target, returns map of
+                                                // SCOM addresses and values
+
+        if (i_opType == DeviceFW::READ && targ != g_scomCache.end())
+        {
+            auto scomVal = targ->second.find(l_addr); // find SCOM address
+
+            if (scomVal != targ->second.end())
+            {
+                // If we found the SCOM in the cache, then erase it from the
+                // cache and return the value.
+                found = true;
+                uint64_t *val = static_cast<uint64_t *>(io_buffer);
+                *val = scomVal->second;
+                io_buflen = sizeof(uint64_t);
+                targ->second.erase(scomVal);
+            }
+        }
+
+        if (i_opType == DeviceFW::WRITE || found == false)
+        {
+            l_err = sendScomOpToFsp(i_opType, i_target, l_addr, io_buffer);
+        }
+    }
+
+    TRACDCOMP(g_trac_fsiscom,EXIT_MRK"fsiScomPerformOp");
+
+    return l_err;
+}
+
+}; // end namespace FSISCOM
diff --git a/src/usr/scom/runtime/rt_scom.C b/src/usr/scom/runtime/rt_scom.C
index 26bac2898..551e9bac9 100644
--- a/src/usr/scom/runtime/rt_scom.C
+++ b/src/usr/scom/runtime/rt_scom.C
@@ -28,6 +28,7 @@
 #include <errl/errlmanager.H>
 #include <scom/scomreasoncodes.H>
 #include <scom/scomif.H>
+#include <scom/runtime/rt_scomif.H>
 #include <runtime/interface.h>
 #include <runtime/rt_targeting.H>
 #include <xscom/piberror.H>
@@ -223,6 +224,13 @@ errlHndl_t sendScomToHyp(DeviceFW::OperationType i_opType,
                                                     i_opType),
                                     i_scomAddr);
 
+                constexpr int MembufFatalError = -0x1008;
+
+                if (rc == MembufFatalError)
+                {
+                    FSISCOM::switchToFspScomAccess(i_target);
+                }
+
                 // attempt to translate rc into a pib error assuming
                 //  the rc is in common format
                 HbrtRcPiberr_t l_commonRc = static_cast<HbrtRcPiberr_t>(rc);
diff --git a/src/usr/scom/runtime/test/testscom_rt.H b/src/usr/scom/runtime/test/testscom_rt.H
index bfa92afde..901b492f8 100644
--- a/src/usr/scom/runtime/test/testscom_rt.H
+++ b/src/usr/scom/runtime/test/testscom_rt.H
@@ -176,7 +176,7 @@ public:
       TRACFCOMP( g_trac_scom, "ScomTest::test_SCOMreadWrite_proc> %d/%d fails", fails, total );
 
   }
-
+      // FSI access in runtime
   /**
    * @brief SCOM test via FSISCOM to Centaur
    *
@@ -186,6 +186,8 @@ public:
       TRACFCOMP( g_trac_scom, "ScomTest::test_FSISCOMreadWrite_centaur> Start" );
       uint64_t fails = 0;
       uint64_t total = 0;
+#if 0 // removing this test from runtime because HB doesn't have any FSI SCOM
+      // access during runtime (except through FSP)
       errlHndl_t l_err = NULL;
 
       // Setup some targets to use
@@ -252,8 +254,10 @@ public:
           {
               continue;
           }
-          else if (scom_targets[x]->
-                getAttr<TARGETING::ATTR_HWAS_STATE>().functional != true)
+          else if ((scom_targets[x]->
+                getAttr<TARGETING::ATTR_HWAS_STATE>().functional != true) ||
+                   (scom_targets[x]->
+                getAttr<TARGETING::ATTR_SCOM_SWITCHES>().useFsiScom == 0))
           {
               TRACDCOMP( g_trac_scom, "ScomTest::test_FSISCOMreadWrite_centaur> Target %d is not functional", x );
               scom_targets[x] = NULL; //remove from our list
@@ -347,6 +351,7 @@ public:
           }
       }
 
+#endif
       TRACFCOMP( g_trac_scom, "ScomTest::test_FSISCOMreadWrite_centaur> %d/%d fails", fails, total );
   }
 
diff --git a/src/usr/util/runtime/rt_cmds.C b/src/usr/util/runtime/rt_cmds.C
index e350e42dc..ce7df1daf 100644
--- a/src/usr/util/runtime/rt_cmds.C
+++ b/src/usr/util/runtime/rt_cmds.C
@@ -43,6 +43,9 @@
 #include <isteps/pm/pm_common_ext.H>
 #include <p9_hcd_memmap_base.H>  // for reload_pm_complex
 #include <p9_stop_data_struct.H> // for reload_pm_complex
+#include <scom/runtime/rt_scomif.H> // sendScomOpToFsp,
+                                    // sendMultiScomReadToFsp,
+                                    // switchToFspScomAccess
 
 extern char hbi_ImageId;
 
@@ -994,6 +997,158 @@ void cmd_hbrt_update(char*& o_output)
 
 
 /**
+ * @brief Mark a target as requiring access to its SCOMs through the FSP
+ * @param[out] o_output     Output display buffer, memory allocated here
+ * @param[in]  i_huid       HUID associated with Target to switch access on
+ */
+void cmd_switchToFspScomAccess( char*& o_output, uint32_t i_huid)
+{
+    UTIL_FT( "cmd_switchToFspScomAccess> huid=%.8X", i_huid );
+
+    TARGETING::Target* l_targ{};
+
+    if(0xFFFFFFFF == i_huid)
+    {
+        TARGETING::targetService().getTopLevelTarget(l_targ);
+    }
+    else
+    {
+        l_targ = getTargetFromHUID(i_huid);
+    }
+
+    o_output = new char[100];
+    if( l_targ == NULL )
+    {
+        sprintf( o_output, "HUID %.8X not found", i_huid );
+        return;
+    }
+
+    FSISCOM::switchToFspScomAccess(l_targ);
+
+    sprintf( o_output, "switchToFspScomAccess executed");
+}
+
+
+/**
+ * @brief Send a scom operation (read/write) to the FSP
+ * @param[out] o_output     Output display buffer, memory allocated here
+ * @param[in]  i_op         Operation: r or w
+ * @param[in]  i_huid       HUID associated with Target to get the SCOM from
+ * @param[in]  i_scomAddr   Address of SCOM to read or write
+ * @param[in]  io_scomValue Buffer for read SCOM value, or value to write
+ */
+void cmd_sendScomOpToFSP( char*&   o_output,
+                          char     i_op,
+                          uint32_t i_huid,
+                          uint64_t i_scomAddr,
+                          uint64_t *io_scomValue )
+{
+    UTIL_FT( "cmd_getScomFromFSP> op=%c, huid=%.8X, addr=%.8X, size=%d",
+             i_op, i_huid, i_scomAddr, *io_scomValue );
+
+    TARGETING::Target* l_targ{};
+
+    if(0xFFFFFFFF == i_huid)
+    {
+        TARGETING::targetService().getTopLevelTarget(l_targ);
+    }
+    else
+    {
+        l_targ = getTargetFromHUID(i_huid);
+    }
+
+    o_output = new char[100];
+    if( l_targ == NULL )
+    {
+        sprintf( o_output, "HUID %.8X not found", i_huid );
+        return;
+    }
+
+    DeviceFW::OperationType l_op;
+    switch (i_op) {
+        case 'r':
+        case 'R':
+            l_op = DeviceFW::READ;
+            break;
+        case 'w':
+        case 'W':
+            l_op = DeviceFW::WRITE;
+            break;
+        default:
+            sprintf( o_output, "Operation must be r or w: %c", i_op );
+            return;
+    }
+
+    errlHndl_t l_err = nullptr;
+    l_err = FSISCOM::sendScomOpToFsp(l_op, l_targ, i_scomAddr,
+                                     (void *)io_scomValue);
+    if (l_err)
+    {
+            sprintf( o_output, "Error on call to sendScomOpToFsp, rc=%.4X",
+                     ERRL_GETRC_SAFE(l_err) );
+            return;
+    }
+
+    sprintf( o_output, "op=%c, huid=%.16llX, scomAddr=%.16llX, scomValue=%.16llX",
+             i_op, i_huid, i_scomAddr, *io_scomValue);
+}
+
+
+/**
+ * @brief Send a multi scom read to the FSP
+ * @param[out] o_output     Output display buffer, memory allocated here
+ * @param[in]  i_huid       HUID associated with Target to get the SCOMs from
+ * @param[in]  i_scomAddr   Addresses of SCOMs to read
+ * @param[in]  o_scomValue  Values of read SCOMs
+ */
+void cmd_sendMultiScomReadToFSP( char*                 &o_output,
+                                 uint32_t               i_huid,
+                                 std::vector<uint64_t> &i_scomAddr,
+                                 std::vector<uint64_t> &o_scomValue )
+{
+    UTIL_FT( "cmd_sendMultiScomReadToFSP> huid=%.8X, num_SCOMs=%d,"
+             " num_outSCOMs=%d",
+             i_huid, i_scomAddr.size(), o_scomValue.size() );
+
+    TARGETING::Target* l_targ{};
+
+    if(0xFFFFFFFF == i_huid)
+    {
+        TARGETING::targetService().getTopLevelTarget(l_targ);
+    }
+    else
+    {
+        l_targ = getTargetFromHUID(i_huid);
+    }
+
+    o_output = new char[500];
+    if( l_targ == NULL )
+    {
+        sprintf( o_output, "HUID %.8X not found", i_huid );
+        return;
+    }
+
+    errlHndl_t l_err = nullptr;
+    l_err = FSISCOM::sendMultiScomReadToFsp( l_targ, i_scomAddr, o_scomValue);
+    if (l_err)
+    {
+            sprintf( o_output, "Error on call to sendMultiScomReadToFsp,"
+                               " rc=%.4X", ERRL_GETRC_SAFE(l_err) );
+            return;
+    }
+
+    sprintf( o_output, "num_outSCOMs=%d", o_scomValue.size());
+    for (auto scom: o_scomValue)
+    {
+        char tmp_str[100];
+
+        sprintf( tmp_str, ", %.8llX", scom);
+        strcat( o_output, tmp_str);
+    }
+}
+
+
+/**
  * @brief  Execute an arbitrary command inside Hostboot Runtime
  * @param[in]   Number of arguments (standard C args)
  * @param[in]   Array of argument values (standard C args)
@@ -1254,9 +1409,67 @@ int hbrtCommand( int argc,
                      "ERROR: hbrt_update\n" );
         }
     }
+    else if( !strcmp( argv[0], "switchToFspScomAccess" ) )
+    {
+        if (argc == 2)
+        {
+            cmd_switchToFspScomAccess( *l_output,
+                                       strtou64(argv[1], NULL, 16)); // huid
+        }
+        else
+        {
+            *l_output = new char[100];
+            sprintf(*l_output,
+                    "ERROR: switchToFspScomAccess <huid>");
+        }
+    }
+    else if( !strcmp( argv[0], "scomOpToFsp" ) )
+    {
+        if ((argc == 4) || (argc == 5))
+        {
+            uint64_t l_scomValue = 0;
+
+            if (argc == 5)
+                l_scomValue = strtou64( argv[4], NULL, 16 );  // value
+
+            cmd_sendScomOpToFSP( *l_output,
+                                 argv[1][0],                  // op
+                                 strtou64(argv[2], NULL, 16), // huid
+                                 strtou64(argv[3], NULL, 16), // addr
+                                 &l_scomValue );
+        }
+        else
+        {
+            *l_output = new char[100];
+            sprintf(*l_output,
+                    "ERROR: scomOpToFsp <op> <huid> <scomAddr> [<scomValue>]");
+        }
+    }
+    else if( !strcmp( argv[0], "multiScomReadToFsp" ) )
+    {
+        if (argc >= 3)
+        {
+            std::vector<uint64_t> l_scomAddrs, l_scomValues;
+
+            for (int i = 2;i < argc;++i)
+                l_scomAddrs.push_back(strtou64( argv[i], NULL, 16 ));
+            l_scomValues.reserve(argc - 2);
+
+            cmd_sendMultiScomReadToFSP( *l_output,
+                                        strtou64(argv[1], NULL, 16), // huid
+                                        l_scomAddrs,
+                                        l_scomValues );
+        }
+        else
+        {
+            *l_output = new char[100];
+            sprintf(*l_output,
+                    "ERROR: multiScomReadToFsp <huid> <scomAddrs>");
+        }
+    }
     else
     {
-        *l_output = new char[50+100*10];
+        *l_output = new char[50+100*12];
         char l_tmpstr[100];
         sprintf( *l_output, "HBRT Commands:\n" );
         sprintf( l_tmpstr, "testRunCommand <args...>\n" );
@@ -1283,6 +1496,13 @@ int hbrtCommand( int argc,
         strcat( *l_output, l_tmpstr );
         sprintf( l_tmpstr, "hbrt_update\n");
         strcat( *l_output, l_tmpstr );
+        sprintf( l_tmpstr, "switchToFspScomAccess <huid>\n");
+        strcat( *l_output, l_tmpstr );
+        sprintf( l_tmpstr, "scomOpToFsp <op> <huid> <scomAddr> [<scomValue>]\n"
+                           "            <op> == r|w\n");
+        strcat( *l_output, l_tmpstr );
+        sprintf( l_tmpstr, "multiScomReadToFsp <huid> <scomAddrs>\n");
+        strcat( *l_output, l_tmpstr );
     }
 
     if( l_traceOut && (*l_output != NULL) )
diff --git a/src/usr/xscom/runtime/rt_xscom.C b/src/usr/xscom/runtime/rt_xscom.C
index 404753bf0..c38f9ef5b 100644
--- a/src/usr/xscom/runtime/rt_xscom.C
+++ b/src/usr/xscom/runtime/rt_xscom.C
@@ -43,6 +43,7 @@ DEVICE_REGISTER_ROUTE(DeviceFW::WILDCARD,
                       TARGETING::TYPE_PROC,
                       xscomPerformOp);
 
+#if 0
 // Also direct fsi scom calls though this interface at runtime
 DEVICE_REGISTER_ROUTE(DeviceFW::WILDCARD,
                       DeviceFW::FSISCOM,
@@ -53,6 +54,7 @@ DEVICE_REGISTER_ROUTE(DeviceFW::WILDCARD,
                       DeviceFW::FSISCOM,
                       TARGETING::TYPE_MEMBUF,
                       xscomPerformOp);
+#endif
 
 
 /**