From 4239927ee5de613ba2127f2788b482ab7de0145c Mon Sep 17 00:00:00 2001 From: Ricardo Mata Date: Thu, 14 Jul 2016 15:02:59 -0500 Subject: p9_suspend_io procedure with updates from review feedback Change-Id: I911f17d72d36ad43fb03cdf86b5ce3d4b07a71fc Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/27218 Tested-by: Jenkins Server Reviewed-by: Jennifer A. Stofer Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/38267 Tested-by: FSP CI Jenkins Reviewed-by: Sachin Gupta --- .../chips/p9/procedures/hwp/nest/p9_suspend_io.C | 299 +++++++++++++++++++++ .../chips/p9/procedures/hwp/nest/p9_suspend_io.H | 90 +++++++ .../xml/error_info/p9_suspend_io_errors.xml | 74 +++++ 3 files changed, 463 insertions(+) create mode 100644 src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C create mode 100644 src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H create mode 100755 src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C new file mode 100644 index 00000000..11fba633 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C @@ -0,0 +1,299 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.C $ */ +/* */ +/* OpenPOWER sbe Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2016,2017 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9_suspend_io.C +/// @brief Procedure to suspend PCIe traffic (FAPI2) +/// +// *HWP HWP Owner: Ricardo Mata Jr. ricmata@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 2 +// *HWP Consumed by: HB + +//----------------------------------------------------------------------------------- +// Includes +//----------------------------------------------------------------------------------- +#include +#include "p9_misc_scom_addresses.H" +#include "p9_misc_scom_addresses_fld.H" + +//---------------------------// +// Function definitions // +//---------------------------// +//------------------------------------------------------------------------------ +// name: p9_suspend_io +//------------------------------------------------------------------------------ +// purpose: +// Freeze PEC and drive reset to PHB to suspend all IO traffic. +// +// parameters: +// 'i_target' is reference to chip target +// 'i_enable_suspend' is Boolean flag indicating whether you're running this procedure during MPIPL (true) or non-MPIPL (false) +// +// returns: +// FAPI_RC_SUCCESS (success, forced PCIe PHBs into reset) +// RC_P9_PBCQ_CQ_NOT_IDLE (error, timed out while waiting for traffic to become idle for PBCQ of PCIe controller) +// (Note: refer to file eclipz/chips/p9/working/procedures/xml/error_info/p9_suspend_io_errors.xml) +// getscom/putscom fapi errors +// fapi error assigned from eCMD function failure +// +//------------------------------------------------------------------------------ +fapi2::ReturnCode p9_suspend_io(const fapi2::Target& i_target, const bool i_enable_mpipl) +{ + FAPI_DBG("Start MPIPL Procedure"); + + bool found_freeze_err; //true: found err that freezes PBCQ; otherwise false + uint32_t l_poll_counter; //Number of iterations while polling for inbound_active and outbound_active + uint32_t l_cur_fir_bit = 0; //Counter use to track freeze bits read. + uint64_t act0_bits; + uint64_t act1_bits; + + fapi2::buffer l_buf = 0; + fapi2::buffer l_buf2 = 0; + auto l_phb_chiplets_vec = i_target.getChildren(); + FAPI_DBG("PHB target vec size: %#x\n", l_phb_chiplets_vec.size()); + uint8_t l_phb_id = 0; + + + //Loop through all PHBs configured + for (auto l_phb_chiplets : l_phb_chiplets_vec) + { + //Get the PHB id + FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_phb_chiplets, l_phb_id)); + + //**************************************************************************************************************** + //STEP #1: nop + //**************************************************************************************************************** + + //**************************************************************************************************************** + //Recov_2: Force PEC into Freeze mode by writing into SW Freeze bit in PCI NEST FIR(26), if PEC not already frozen. + //**************************************************************************************************************** + + //Initially, start with no error found that freezes PEC + found_freeze_err = false; + + FAPI_DBG("PHB%i: Recov_2 - Force freeze the PEC if not already frozen.", l_phb_id); + + //Read PCI Nest FIR Action0 register and put contents into l_buf + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIRACTION0_REG, l_buf)); + FAPI_DBG("PHB%i: PCI Nest FIR Action0 Register %#lx", l_phb_id, l_buf()); + act0_bits = l_buf; + + //Read PCI Nest FIR Action1 register and put contents into l_buf + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIRACTION1_REG, l_buf)); + FAPI_DBG("PHB%i: PCI Nest FIR Action1 Register %#lx", l_phb_id, l_buf()); + act1_bits = l_buf; + + //AND Action0 and Action1 bits to determine which FIR bits are set up for Freeze = 0b11 + l_buf2 = act0_bits & act1_bits; + FAPI_DBG("PHB%i: PCI Nest FIR Freeze Bits %#lx", l_phb_id, l_buf2()); + + //Read PCI Nest FIR register and put contents into l_buf + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf)); + FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf()); + + //Inspect which FIR bits are set AND if action is set to Freeze + while (l_cur_fir_bit < PHB_NFIR_REG_NFIRNFIR_LEN) + { + if(l_buf.getBit(l_cur_fir_bit) && l_buf2.getBit(l_cur_fir_bit)) + { + FAPI_DBG("PHB%i: PEC is already frozen. PCI Nest FIR[%u] is asserted.", l_phb_id, l_cur_fir_bit); + found_freeze_err = true; + break; + } + + l_cur_fir_bit++; + } + + //Found PEC to not be frozen + if(!found_freeze_err) + { + FAPI_DBG("PHB%i: PEC not frozen. Forcing PEC into freeze mode.", l_phb_id); + + //Force PEC freeze by setting SW Freeze bit in PCI Nest FIR Register + l_buf = 0; + l_buf.setBit(); + FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf()); + FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_NFIR_REG_OR, l_buf)); + } + + FAPI_DBG("PHB%i: Successfully put PEC into freeze state", l_phb_id); + + //**************************************************************************************************************** + //Recov_3: Put the PHB in reset state + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_3 - Put PHB into reset state.", l_phb_id); + + //Set bit0 of the PHB Reset Register to drive reset active to the phb + l_buf = 0; + l_buf.setBit(); + FAPI_DBG("PHB%i: PHB Reset Register %#lx", l_phb_id, l_buf()); + FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_PHBRESET_REG, l_buf)); + + FAPI_DBG("PHB%i: Successfully put PHB into reset", l_phb_id); + + //**************************************************************************************************************** + //Recov_4: Poll the CQ Status until all actively working DMA/CI operations are done by waiting for bits[0:1] = 00 + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_4 - Start polling for inbound_active and outbound_active state machines to become idle.", + l_phb_id); + l_poll_counter = 0; //Reset poll counter + + while (l_poll_counter < MAX_NUM_POLLS) + { + l_poll_counter++; + FAPI_TRY(fapi2::delay(PBCQ_NANO_SEC_DELAY, PBCQ_SIM_CYC_DELAY), "fapiDelay error."); + + //Read PBCQ General Status Register and put contents into l_buf + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_CQSTAT_REG, l_buf), "Could not retrieve PHB_CQSTAT_REG."); + FAPI_DBG("PHB%i: PBCQ General Status Register %#lx", l_phb_id, l_buf()); + + //Check for bits 0 (inbound_active) and 1 (outbound_active) to become deasserted + if (!(l_buf.getBit(PEC_STACK0_CQSTAT_REG_PE_INBOUND_ACTIVE) || l_buf.getBit(PEC_STACK0_CQSTAT_REG_PE_OUTBOUND_ACTIVE))) + { + + FAPI_DBG("PHB%i: PBCQ CQ status is idle.", l_phb_id); + FAPI_DBG("PHB%i: End polling for inbound_active and outbound_active state machines to become idle.", l_phb_id); + break; + } + } + + FAPI_DBG("PHB%i: inbound_active and outbound_active status (poll counter = %d).", l_phb_id, l_poll_counter); + fapi2::buffer l_buf3 = 0; + + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf2)); + FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf2()); + + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PHBRESET_REG, l_buf3)); + FAPI_DBG("PHB%i: PHB Reset Register %#lx", l_phb_id, l_buf3()); + + FAPI_ASSERT(l_poll_counter < MAX_NUM_POLLS, + fapi2::P9_PBCQ_CQ_NOT_IDLE() + .set_TARGET(l_phb_chiplets) + .set_NFIR_ADDR(PHB_NFIR_REG) + .set_NFIR_DATA(l_buf2) + .set_PHB_RESET_ADDR(PHB_PHBRESET_REG) + .set_PHB_RESET_DATA(l_buf3) + .set_CQ_STAT_ADDR(PHB_CQSTAT_REG) + .set_CQ_STAT_DATA(l_buf), + "PHB%i: PBCQ CQ Status did not clear. It should!", l_phb_id); + + + //**************************************************************************************************************** + //Recov_5: Clear the BAR and IRSN enables to avoid addressing conflict post IPL. + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_5 - Clearing Base Address Enabled register.", l_phb_id); + + //Clear BARE register + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_BARE_REG, l_buf)); + l_buf.clearBit() + .clearBit() + .clearBit() + .clearBit(); + FAPI_DBG("PHB%i: Base Address Enable Register %#lx", l_phb_id, l_buf()); + FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_BARE_REG, l_buf)); + + FAPI_DBG("PHB%i: Successfully cleared BARE", l_phb_id); + + //**************************************************************************************************************** + //Recov_6: Clear PCI FIR (PFIR) Freeze Errors. + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_6 - Clearing PCI FIR.", l_phb_id); + + //Clear FIR bits of PCI FIR register + l_buf = (uint64_t)0x0; + FAPI_DBG("PHB%i: PCI FIR Register Clear %#lx", l_phb_id, l_buf()); + FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_PFIR_REG_AND, l_buf)); + + //Confirm FIR bits have been cleared + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PFIR_REG, l_buf)); + FAPI_DBG("PHB%i: PCI FIR Register %#lx", l_phb_id, l_buf()); + + FAPI_DBG("PHB%i: Successfully cleared PCI FIR.", l_phb_id); + + //**************************************************************************************************************** + //Recov_7: Clear CERR Report 0/1 bits + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_7 - Clearing CERR Report 0/1 bits.", l_phb_id); + + // TODO: HW363246 this step appears in the v1.0 doc but failed PCB address + + //Clear CERR Report 0/1 bits + //l_buf = (uint64_t)0x0; + //FAPI_DBG("PHB%i: CERR Report0 Register Clear %#lx", l_phb_id, l_buf()); + //FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_CERR_RPT0_REG, l_buf)); + + //l_buf = (uint64_t)0x0; + //FAPI_DBG("PHB%i: CERR Report1 Register Clear %#lx", l_phb_id, l_buf()); + //FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_CERR_RPT1_REG, l_buf)); + + // TODO: HW363246 this step appears in the v1.0 doc but failed PCB address + + FAPI_DBG("PHB%i: Succesfully cleared CERR Report 0/1 bits.", l_phb_id); + + //**************************************************************************************************************** + //Recov_8: Clear Nest FIR (NFIR) Freeze Errors. + //**************************************************************************************************************** + FAPI_DBG("PHB%i: Recov_8 - Clearing PCI Nest FIR.", l_phb_id); + + //Clear FIR bits of PCI Nest FIR register + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf)); + FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf()); + l_buf.invert(); + FAPI_DBG("PHB%i: PCI Nest FIR Register Clear %#lx", l_phb_id, l_buf()); + FAPI_TRY(fapi2::putScom(l_phb_chiplets, PHB_NFIR_REG_AND, l_buf)); + + //Confirm FIR bits have been cleared + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_NFIR_REG, l_buf)); + FAPI_DBG("PHB%i: PCI Nest FIR Register %#lx", l_phb_id, l_buf()); + + FAPI_TRY(fapi2::getScom(l_phb_chiplets, PHB_PFIR_REG, l_buf2)); + FAPI_DBG("PHB%i: PCI FIR Register %#lx", l_phb_id, l_buf2()); + + if (l_buf.getBit()) + { + FAPI_ASSERT(false, + fapi2::P9_PHB_NFIR_NOT_CLEARED() + .set_TARGET(l_phb_chiplets) + .set_NFIR_ADDR(PHB_NFIR_REG) + .set_NFIR_DATA(l_buf) + .set_PFIR_ADDR(PHB_PFIR_REG) + .set_PFIR_DATA(l_buf2), + "PHB%i: PCI Nest FIR Register did not clear. It should!", l_phb_id); + } + + + FAPI_DBG("PHB%i: Succesfully cleared PCI Nest FIR.", l_phb_id); + + //**************************************************************************************************************** + //Recov_9: PEC is ready for firmware to take PHB out of reset and start initialization. + //**************************************************************************************************************** + + } + +fapi_try_exit: + FAPI_DBG("End MPIPL Procedure"); + return fapi2::current_err; +} diff --git a/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H new file mode 100644 index 00000000..697c5410 --- /dev/null +++ b/src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H @@ -0,0 +1,90 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/import/chips/p9/procedures/hwp/nest/p9_suspend_io.H $ */ +/* */ +/* OpenPOWER sbe Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2016,2017 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/// +/// @file p9_suspend_io.H +/// @brief Procedure to suspend PCIe traffic. (FAPI2) +/// +// *HWP HWP Owner: Ricardo Mata Jr. ricmata@us.ibm.com +// *HWP FW Owner: Thi Tran thi@us.ibm.com +// *HWP Team: Nest +// *HWP Level: 2 +// *HWP Consumed by: HB + +#ifndef _P9_SUSPEND_IO_H_ +#define _P9_SUSPEND_IO_H_ + +//----------------------------------------------------------------------------------- +// Includes +//----------------------------------------------------------------------------------- +#include + +//----------------------------------------------------------------------------------- +// Structure definitions +//----------------------------------------------------------------------------------- + +//function pointer typedef definition for HWP call support +typedef fapi2::ReturnCode (*p9_suspend_io_FP_t) (const fapi2::Target&, const bool); + +//----------------------------------------------------------------------------------- +// Constant definitions +//----------------------------------------------------------------------------------- + +const uint8_t PHB_NFIR_REG_SW_DEFINED_FREEZE = 26; //PCI Nest FIR Register bit26 for SW Freeze enablement + +const uint32_t MAX_NUM_POLLS = 100; //Maximum number of iterations (So, 20ms * 100 = 2s before timeout) +const uint64_t PBCQ_NANO_SEC_DELAY = + 40000000; //(40000000 ns = 20 ms) to wait for PBCQ inbound and outbound to become inactive +const uint64_t PBCQ_SIM_CYC_DELAY = + 400000; //400,000 simulation cycles to wait for PBCQ inbound and outbound to become inactive + + +extern "C" { + +//----------------------------------------------------------------------------------- +// Function prototype +//----------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ +// name: p9_suspend_io +//------------------------------------------------------------------------------ +// purpose: +// Freeze PEC and drive reset to PHB to suspend all IO traffic. +// +// parameters: +// 'i_target' is reference to chip target +// 'i_enable_mpipl' is Boolean flag indicating whether you're running this procedure during MPIPL (true) or non-MPIPL (false) +// +// returns: +// FAPI_RC_SUCCESS (success, forced PCIe PHBs into reset) +// RC_P9_PBCQ_CQ_NOT_IDLE (error, timed out while waiting for traffic to become idle for PBCQ of PCIe controller) +// (Note: refer to file eclipz/chips/p9/working/procedures/xml/error_info/p9_suspend_io_errors.xml) +// getscom/putscom fapi errors +// fapi error assigned from eCMD function failure +// +//------------------------------------------------------------------------------ + fapi2::ReturnCode p9_suspend_io(const fapi2::Target& i_target, const bool i_enable_mpipl); + +} //extern"C" + +#endif //_P9_SUSPEND_IO_H_ diff --git a/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml b/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml new file mode 100755 index 00000000..0f4bdbc0 --- /dev/null +++ b/src/import/chips/p9/procedures/xml/error_info/p9_suspend_io_errors.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RC_P9_PBCQ_CQ_NOT_IDLE + + PBCQ CQ did not go to idle. + + TARGET + NFIR_ADDR + NFIR_DATA + PHB_RESET_ADDR + PHB_RESET_DATA + CQ_STAT_ADDR + CQ_STAT_DATA + + + CODE + LOW + + + + TARGET + HIGH + + + + + RC_P9_PHB_NFIR_NOT_CLEARED + + PCI Nest FIR Register did not clear. + + TARGET + NFIR_ADDR + NFIR_DATA + PFIR_ADDR + PFIR_DATA + + + CODE + LOW + + + + TARGET + HIGH + + + + -- cgit v1.2.1