p9_suspend_io procedure with updates from review feedback

Change-Id: I911f17d72d36ad43fb03cdf86b5ce3d4b07a71fc
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/27218
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/38267
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>

3 files changed, 463 insertions, 0 deletions

diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C
new file mode 100644
index 00000000..11fba633
--- /dev/null
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C
@@ -0,0 +1,299 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C $     */
+/*                                                                        */
+/* OpenPOWER sbe Project                                                  */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+///
+/// @file p9_suspend_io.C
+/// @brief Procedure to suspend PCIe traffic (FAPI2)
+///
+// *HWP HWP Owner: Ricardo Mata Jr. ricmata@us.ibm.com
+// *HWP FW Owner: Thi Tran thi@us.ibm.com
+// *HWP Team: Nest
+// *HWP Level: 2
+// *HWP Consumed by: HB
+
+//-----------------------------------------------------------------------------------
+// Includes
+//-----------------------------------------------------------------------------------
+#include <p9_suspend_io.H>
+#include "p9_misc_scom_addresses.H"
+#include "p9_misc_scom_addresses_fld.H"
+
+//---------------------------//
+// Function definitions      //
+//---------------------------//
+//------------------------------------------------------------------------------
+// name: p9_suspend_io
+//------------------------------------------------------------------------------
+// purpose:
+// Freeze PEC and drive reset to PHB to suspend all IO traffic.
+//
+// parameters:
+// 'i_target' is reference to chip target
+// 'i_enable_suspend' is Boolean flag indicating whether you're running this procedure during MPIPL (true) or non-MPIPL (false)
+//
+// returns:
+// FAPI_RC_SUCCESS (success, forced PCIe PHBs into reset)
+// RC_P9_PBCQ_CQ_NOT_IDLE (error, timed out while waiting for traffic to become idle for PBCQ of PCIe controller)
+// (Note: refer to file eclipz/chips/p9/working/procedures/xml/error_info/p9_suspend_io_errors.xml)
+// getscom/putscom fapi errors
+// fapi error assigned from eCMD function failure
+//
+//------------------------------------------------------------------------------
+fapi2::ReturnCode p9_suspend_io(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, const bool i_enable_mpipl)
+{
+    FAPI_DBG("Start MPIPL Procedure");
+
+    bool found_freeze_err; //true: found err that freezes PBCQ; otherwise false
+    uint32_t l_poll_counter; //Number of iterations while polling for inbound_active and outbound_active
+    uint32_t l_cur_fir_bit = 0; //Counter use to track freeze bits read.
+    uint64_t act0_bits;
+    uint64_t act1_bits;
+
+    fapi2::buffer<uint64_t> l_buf = 0;
+    fapi2::buffer<uint64_t> l_buf2 = 0;
+    auto l_phb_chiplets_vec = i_target.getChildren<fapi2::TARGET_TYPE_PHB>();
+    FAPI_DBG("PHB target vec size: %#x\n", l_phb_chiplets_vec.size());
+    uint8_t l_phb_id = 0;
+
+
+    //Loop through all PHBs configured
+    for (auto l_phb_chiplets : l_phb_chiplets_vec)
+    {
+        //Get the PHB id
+        FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_phb_chiplets, l_phb_id));
+
+        //****************************************************************************************************************
+        //STEP #1: nop
+        //****************************************************************************************************************
+
+        //****************************************************************************************************************
+        //Recov_2: Force PEC into Freeze mode by writing into SW Freeze bit in PCI NEST FIR(26), if PEC not already frozen.
+        //****************************************************************************************************************
+
+        //Initially, start with no error found that freezes PEC
+        found_freeze_err = false;
+
+        FAPI_DBG("PHB%i: Recov_2 - Force freeze the PEC if not already frozen.", l_phb_id);
+
+        //Read PCI Nest FIR Action0 register and put contents into l_buf
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIRACTION0_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI Nest FIR Action0 Register %#lx", l_phb_id, l_buf());
+        act0_bits = l_buf;
+
+        //Read PCI Nest FIR Action1 register and put contents into l_buf
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIRACTION1_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI Nest FIR Action1 Register %#lx", l_phb_id, l_buf());
+        act1_bits = l_buf;
+
+        //AND Action0 and Action1 bits to determine which FIR bits are set up for Freeze = 0b11
+        l_buf2 = act0_bits & act1_bits;
+        FAPI_DBG("PHB%i: PCI Nest FIR Freeze Bits %#lx", l_phb_id, l_buf2());
+
+        //Read PCI Nest FIR register and put contents into l_buf
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf());
+
+        //Inspect which FIR bits are set AND if action is set to Freeze
+        while (l_cur_fir_bit < PHB_NFIR_REG_NFIRNFIR_LEN)
+        {
+            if(l_buf.getBit(l_cur_fir_bit) && l_buf2.getBit(l_cur_fir_bit))
+            {
+                FAPI_DBG("PHB%i: PEC is already frozen. PCI Nest FIR[%u] is asserted.", l_phb_id, l_cur_fir_bit);
+                found_freeze_err = true;
+                break;
+            }
+
+            l_cur_fir_bit++;
+        }
+
+        //Found PEC to not be frozen
+        if(!found_freeze_err)
+        {
+            FAPI_DBG("PHB%i: PEC not frozen. Forcing PEC into freeze mode.", l_phb_id);
+
+            //Force PEC freeze by setting SW Freeze bit in PCI Nest FIR Register
+            l_buf = 0;
+            l_buf.setBit<PHB_NFIR_REG_SW_DEFINED_FREEZE>();
+            FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf());
+            FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_NFIR_REG_OR, l_buf));
+        }
+
+        FAPI_DBG("PHB%i: Successfully put PEC into freeze state", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_3: Put the PHB in reset state
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_3 - Put PHB into reset state.", l_phb_id);
+
+        //Set bit0 of the PHB Reset Register to drive reset active to the phb
+        l_buf = 0;
+        l_buf.setBit<PHB_PHBRESET_REG_PE_ETU_RESET>();
+        FAPI_DBG("PHB%i: PHB Reset Register %#lx", l_phb_id, l_buf());
+        FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_PHBRESET_REG, l_buf));
+
+        FAPI_DBG("PHB%i: Successfully put PHB into reset", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_4: Poll the CQ Status until all actively working DMA/CI operations are done by waiting for bits[0:1] = 00
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_4 - Start polling for inbound_active and outbound_active state machines to become idle.",
+                 l_phb_id);
+        l_poll_counter = 0; //Reset poll counter
+
+        while (l_poll_counter < MAX_NUM_POLLS)
+        {
+            l_poll_counter++;
+            FAPI_TRY(fapi2::delay(PBCQ_NANO_SEC_DELAY, PBCQ_SIM_CYC_DELAY), "fapiDelay error.");
+
+            //Read PBCQ General Status Register and put contents into l_buf
+            FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_CQSTAT_REG, l_buf), "Could not retrieve PHB_CQSTAT_REG.");
+            FAPI_DBG("PHB%i: PBCQ General Status Register %#lx", l_phb_id, l_buf());
+
+            //Check for bits 0 (inbound_active) and 1 (outbound_active) to become deasserted
+            if (!(l_buf.getBit(PEC_STACK0_CQSTAT_REG_PE_INBOUND_ACTIVE) || l_buf.getBit(PEC_STACK0_CQSTAT_REG_PE_OUTBOUND_ACTIVE)))
+            {
+
+                FAPI_DBG("PHB%i: PBCQ CQ status is idle.", l_phb_id);
+                FAPI_DBG("PHB%i: End polling for inbound_active and outbound_active state machines to become idle.", l_phb_id);
+                break;
+            }
+        }
+
+        FAPI_DBG("PHB%i: inbound_active and outbound_active status (poll counter = %d).", l_phb_id, l_poll_counter);
+        fapi2::buffer<uint64_t> l_buf3 = 0;
+
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf2));
+        FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf2());
+
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PHBRESET_REG, l_buf3));
+        FAPI_DBG("PHB%i: PHB Reset Register %#lx", l_phb_id, l_buf3());
+
+        FAPI_ASSERT(l_poll_counter < MAX_NUM_POLLS,
+                    fapi2::P9_PBCQ_CQ_NOT_IDLE()
+                    .set_TARGET(l_phb_chiplets)
+                    .set_NFIR_ADDR(PHB_NFIR_REG)
+                    .set_NFIR_DATA(l_buf2)
+                    .set_PHB_RESET_ADDR(PHB_PHBRESET_REG)
+                    .set_PHB_RESET_DATA(l_buf3)
+                    .set_CQ_STAT_ADDR(PHB_CQSTAT_REG)
+                    .set_CQ_STAT_DATA(l_buf),
+                    "PHB%i: PBCQ CQ Status did not clear. It should!", l_phb_id);
+
+
+        //****************************************************************************************************************
+        //Recov_5: Clear the BAR and IRSN enables to avoid addressing conflict post IPL.
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_5 - Clearing Base Address Enabled register.", l_phb_id);
+
+        //Clear BARE register
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_BARE_REG, l_buf));
+        l_buf.clearBit<PHB_BARE_REG_PE_MMIO_BAR0_EN>()
+        .clearBit<PHB_BARE_REG_PE_MMIO_BAR1_EN>()
+        .clearBit<PHB_BARE_REG_PE_PHB_BAR_EN>()
+        .clearBit<PHB_BARE_REG_PE_INT_BAR_EN>();
+        FAPI_DBG("PHB%i: Base Address Enable Register %#lx", l_phb_id, l_buf());
+        FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_BARE_REG, l_buf));
+
+        FAPI_DBG("PHB%i: Successfully cleared BARE", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_6: Clear PCI FIR (PFIR) Freeze Errors.
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_6 - Clearing PCI FIR.", l_phb_id);
+
+        //Clear FIR bits of PCI FIR register
+        l_buf = (uint64_t)0x0;
+        FAPI_DBG("PHB%i: PCI FIR Register Clear %#lx", l_phb_id, l_buf());
+        FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_PFIR_REG_AND, l_buf));
+
+        //Confirm FIR bits have been cleared
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PFIR_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI FIR Register %#lx", l_phb_id, l_buf());
+
+        FAPI_DBG("PHB%i: Successfully cleared PCI FIR.", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_7: Clear CERR Report 0/1 bits
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_7 - Clearing CERR Report 0/1 bits.", l_phb_id);
+
+        // TODO: HW363246 this step appears in the v1.0 doc but failed PCB address
+
+        //Clear CERR Report 0/1 bits
+        //l_buf = (uint64_t)0x0;
+        //FAPI_DBG("PHB%i: CERR Report0 Register Clear %#lx", l_phb_id, l_buf());
+        //FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_CERR_RPT0_REG, l_buf));
+
+        //l_buf = (uint64_t)0x0;
+        //FAPI_DBG("PHB%i: CERR Report1 Register Clear %#lx", l_phb_id, l_buf());
+        //FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_CERR_RPT1_REG, l_buf));
+
+        // TODO: HW363246 this step appears in the v1.0 doc but failed PCB address
+
+        FAPI_DBG("PHB%i: Succesfully cleared CERR Report 0/1 bits.", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_8: Clear Nest FIR (NFIR) Freeze Errors.
+        //****************************************************************************************************************
+        FAPI_DBG("PHB%i: Recov_8 - Clearing PCI Nest FIR.", l_phb_id);
+
+        //Clear FIR bits of PCI Nest FIR register
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf());
+        l_buf.invert();
+        FAPI_DBG("PHB%i: PCI Nest FIR Register Clear %#lx", l_phb_id, l_buf());
+        FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_NFIR_REG_AND, l_buf));
+
+        //Confirm FIR bits have been cleared
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf));
+        FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf());
+
+        FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PFIR_REG, l_buf2));
+        FAPI_DBG("PHB%i: PCI FIR Register %#lx", l_phb_id, l_buf2());
+
+        if (l_buf.getBit<PHB_NFIR_REG_NFIRNFIR, PHB_NFIR_REG_NFIRNFIR_LEN>())
+        {
+            FAPI_ASSERT(false,
+                        fapi2::P9_PHB_NFIR_NOT_CLEARED()
+                        .set_TARGET(l_phb_chiplets)
+                        .set_NFIR_ADDR(PHB_NFIR_REG)
+                        .set_NFIR_DATA(l_buf)
+                        .set_PFIR_ADDR(PHB_PFIR_REG)
+                        .set_PFIR_DATA(l_buf2),
+                        "PHB%i: PCI Nest FIR Register did not clear. It should!", l_phb_id);
+        }
+
+
+        FAPI_DBG("PHB%i: Succesfully cleared PCI Nest FIR.", l_phb_id);
+
+        //****************************************************************************************************************
+        //Recov_9: PEC is ready for firmware to take PHB out of reset and start initialization.
+        //****************************************************************************************************************
+
+    }
+
+fapi_try_exit:
+    FAPI_DBG("End MPIPL Procedure");
+    return fapi2::current_err;
+}
diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H
new file mode 100644
index 00000000..697c5410
--- /dev/null
+++ b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H
@@ -0,0 +1,90 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H $     */
+/*                                                                        */
+/* OpenPOWER sbe Project                                                  */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+///
+/// @file p9_suspend_io.H
+/// @brief Procedure to suspend PCIe traffic. (FAPI2)
+///
+// *HWP HWP Owner: Ricardo Mata Jr. ricmata@us.ibm.com
+// *HWP FW Owner: Thi Tran thi@us.ibm.com
+// *HWP Team: Nest
+// *HWP Level: 2
+// *HWP Consumed by: HB
+
+#ifndef _P9_SUSPEND_IO_H_
+#define _P9_SUSPEND_IO_H_
+
+//-----------------------------------------------------------------------------------
+// Includes
+//-----------------------------------------------------------------------------------
+#include <fapi2.H>
+
+//-----------------------------------------------------------------------------------
+// Structure definitions
+//-----------------------------------------------------------------------------------
+
+//function pointer typedef definition for HWP call support
+typedef fapi2::ReturnCode (*p9_suspend_io_FP_t) (const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>&, const bool);
+
+//-----------------------------------------------------------------------------------
+// Constant definitions
+//-----------------------------------------------------------------------------------
+
+const uint8_t PHB_NFIR_REG_SW_DEFINED_FREEZE = 26; //PCI Nest FIR Register bit26 for SW Freeze enablement
+
+const uint32_t MAX_NUM_POLLS = 100; //Maximum number of iterations (So, 20ms * 100 = 2s before timeout)
+const uint64_t PBCQ_NANO_SEC_DELAY =
+    40000000; //(40000000 ns = 20 ms) to wait for PBCQ inbound and outbound to become inactive
+const uint64_t PBCQ_SIM_CYC_DELAY =
+    400000; //400,000 simulation cycles to wait for PBCQ inbound and outbound to become inactive
+
+
+extern "C" {
+
+//-----------------------------------------------------------------------------------
+// Function prototype
+//-----------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+// name: p9_suspend_io
+//------------------------------------------------------------------------------
+// purpose:
+// Freeze PEC and drive reset to PHB to suspend all IO traffic.
+//
+// parameters:
+// 'i_target' is reference to chip target
+// 'i_enable_mpipl' is Boolean flag indicating whether you're running this procedure during MPIPL (true) or non-MPIPL (false)
+//
+// returns:
+// FAPI_RC_SUCCESS (success, forced PCIe PHBs into reset)
+// RC_P9_PBCQ_CQ_NOT_IDLE (error, timed out while waiting for traffic to become idle for PBCQ of PCIe controller)
+// (Note: refer to file eclipz/chips/p9/working/procedures/xml/error_info/p9_suspend_io_errors.xml)
+// getscom/putscom fapi errors
+// fapi error assigned from eCMD function failure
+//
+//------------------------------------------------------------------------------
+    fapi2::ReturnCode p9_suspend_io(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target, const bool i_enable_mpipl);
+
+} //extern"C"
+
+#endif //_P9_SUSPEND_IO_H_
diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml
new file mode 100755
index 00000000..0f4bdbc0
--- /dev/null
+++ b/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml
@@ -0,0 +1,74 @@
+<!-- IBM_PROLOG_BEGIN_TAG                                                   -->
+<!-- This is an automatically generated prolog.                             -->
+<!--                                                                        -->
+<!-- $Source: src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml $ -->
+<!--                                                                        -->
+<!-- OpenPOWER sbe Project                                                  -->
+<!--                                                                        -->
+<!-- Contributors Listed Below - COPYRIGHT 2016,2017                        -->
+<!-- [+] International Business Machines Corp.                              -->
+<!--                                                                        -->
+<!--                                                                        -->
+<!-- Licensed under the Apache License, Version 2.0 (the "License");        -->
+<!-- you may not use this file except in compliance with the License.       -->
+<!-- You may obtain a copy of the License at                                -->
+<!--                                                                        -->
+<!--     http://www.apache.org/licenses/LICENSE-2.0                         -->
+<!--                                                                        -->
+<!-- Unless required by applicable law or agreed to in writing, software    -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS,      -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        -->
+<!-- implied. See the License for the specific language governing           -->
+<!-- permissions and limitations under the License.                         -->
+<!--                                                                        -->
+<!-- IBM_PROLOG_END_TAG                                                     -->
+<!-- Error definitions for p9_suspend_io procedure -->
+<hwpErrors>
+  <!-- ********************************************************************* -->
+  <hwpError>
+    <rc>RC_P9_PBCQ_CQ_NOT_IDLE</rc>
+    <description>
+      PBCQ CQ did not go to idle.
+    </description>
+    <ffdc>TARGET</ffdc>
+    <ffdc>NFIR_ADDR</ffdc>
+    <ffdc>NFIR_DATA</ffdc>
+    <ffdc>PHB_RESET_ADDR</ffdc>
+    <ffdc>PHB_RESET_DATA</ffdc>
+    <ffdc>CQ_STAT_ADDR</ffdc>
+    <ffdc>CQ_STAT_DATA</ffdc>
+    <!-- Add procedure callout -->
+    <callout>
+      <procedure>CODE</procedure>
+      <priority>LOW</priority>
+    </callout>
+    <!-- Add hw callout -->
+    <callout>
+      <target>TARGET</target>
+      <priority>HIGH</priority>
+    </callout>
+  </hwpError>
+  <!-- ********************************************************************* -->
+  <hwpError>
+    <rc>RC_P9_PHB_NFIR_NOT_CLEARED</rc>
+    <description>
+      PCI Nest FIR Register did not clear.
+    </description>
+    <ffdc>TARGET</ffdc>
+    <ffdc>NFIR_ADDR</ffdc>
+    <ffdc>NFIR_DATA</ffdc>
+    <ffdc>PFIR_ADDR</ffdc>
+    <ffdc>PFIR_DATA</ffdc>
+    <!-- Add procedure callout -->
+    <callout>
+      <procedure>CODE</procedure>
+      <priority>LOW</priority>
+    </callout>
+    <!-- Add hw callout -->
+    <callout>
+      <target>TARGET</target>
+      <priority>HIGH</priority>
+    </callout>
+  </hwpError>
+  <!-- ********************************************************************* -->
+</hwpErrors>