summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristian Geddes <crgeddes@us.ibm.com>2019-04-05 15:58:40 -0500
committerDaniel M. Crowell <dcrowell@us.ibm.com>2019-04-18 10:38:28 -0500
commite583424484de98e8be66549370fb28453ccb98e4 (patch)
tree4485d1d5dd18f91c0db72af79e33f1203f75312f
parent9d139bd7d99a87c6d11f6569482218badb49cc8b (diff)
downloadblackbird-hostboot-e583424484de98e8be66549370fb28453ccb98e4.tar.gz
blackbird-hostboot-e583424484de98e8be66549370fb28453ccb98e4.zip
Validate OMI INBAND BAR offset attributes against calculated values
While setting up the virtual memory mapped IO to the OCMB chips we make some assumptions that the OCMB MMIO spaces will be contiguous. The p9a_omi_setup_bars HWP uses OMI_INBAND_BAR_BASE_ADDR_OFFSET to set the scom registers that determine the physical offset mapped to the IO. When setting up the Virtual addresses hostboot uses to represent the physical mmio address, we must validate that the attribute matches with what we calculated. While doing this we found that the virtual address attribute was being calculated incorrectly. It was not localizing the OCMB position relative to the MC which is required when calculating the offset into the MC bar. Change-Id: I0ebbcd38e19a238e2cc16791bb0595536788bb7f RTC: 201493 Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/75631 Reviewed-by: Matthew Raybuck <matthew.raybuck@ibm.com> Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com> Reviewed-by: Roland Veloz <rveloz@us.ibm.com> Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com> Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com> Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com> Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com> Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
-rw-r--r--src/include/arch/memorymap.H3
-rw-r--r--src/include/usr/mmio/mmio_reasoncodes.H3
-rw-r--r--src/usr/ibscom/ibscom.C5
-rw-r--r--src/usr/mmio/mmio.C214
-rw-r--r--src/usr/targeting/common/xmltohb/simics_AXONE.system.xml12
5 files changed, 122 insertions, 115 deletions
diff --git a/src/include/arch/memorymap.H b/src/include/arch/memorymap.H
index a25bcb62d..7b75f6363 100644
--- a/src/include/arch/memorymap.H
+++ b/src/include/arch/memorymap.H
@@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
-/* Contributors Listed Below - COPYRIGHT 2017,2018 */
+/* Contributors Listed Below - COPYRIGHT 2017,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
@@ -38,6 +38,7 @@
constexpr uint64_t MMIO_OFFSET_PER_CHIP = (4*TERABYTE); //0x40000000000
constexpr uint64_t MMIO_OFFSET_PER_GROUP = (32*TERABYTE); //0x200000000000
+constexpr uint64_t MMIO_BASE = 0x6000000000000;
/**
* @brief Compute MMIO value for a given chip and base value
*/
diff --git a/src/include/usr/mmio/mmio_reasoncodes.H b/src/include/usr/mmio/mmio_reasoncodes.H
index 7e2481934..86ff60b5b 100644
--- a/src/include/usr/mmio/mmio_reasoncodes.H
+++ b/src/include/usr/mmio/mmio_reasoncodes.H
@@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
-/* Contributors Listed Below - COPYRIGHT 2011,2018 */
+/* Contributors Listed Below - COPYRIGHT 2011,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
@@ -51,6 +51,7 @@ namespace MMIO
RC_BAD_MMIO_READ = MMIO_COMP_ID | 0x08,
RC_BAD_MMIO_WRITE = MMIO_COMP_ID | 0x09,
RC_PROC_NOT_FOUND = MMIO_COMP_ID | 0x0A,
+ RC_BAR_OFFSET_MISMATCH = MMIO_COMP_ID | 0x0B,
};
};
diff --git a/src/usr/ibscom/ibscom.C b/src/usr/ibscom/ibscom.C
index e3cb6d468..8939454a3 100644
--- a/src/usr/ibscom/ibscom.C
+++ b/src/usr/ibscom/ibscom.C
@@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
-/* Contributors Listed Below - COPYRIGHT 2012,2018 */
+/* Contributors Listed Below - COPYRIGHT 2012,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
@@ -66,7 +66,6 @@ namespace IBSCOM
// SCOM Register addresses
const uint32_t MBS_FIR = 0x02011400;
const uint32_t MBSIBERR0 = 0x0201141B;
-const uint64_t IBSCOM_BASE = 0x0006000000000000;
const uint64_t BIT_18_MASK = 0x0000000000002000;
// Register XSCcom access functions to DD framework
@@ -217,7 +216,7 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
uint8_t l_chipId =
l_parentChip->getAttr<ATTR_PROC_EFF_FABRIC_CHIP_ID>();
- l_IBScomAddr = computeMemoryMapOffset( IBSCOM_BASE,
+ l_IBScomAddr = computeMemoryMapOffset( MMIO_BASE,
l_groupId,
l_chipId );
l_IBScomAddr = l_IBScomAddr + l_IBScomBaseAddrOffset;
diff --git a/src/usr/mmio/mmio.C b/src/usr/mmio/mmio.C
index cba75bb7c..1b84660f8 100644
--- a/src/usr/mmio/mmio.C
+++ b/src/usr/mmio/mmio.C
@@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
-/* Contributors Listed Below - COPYRIGHT 2018 */
+/* Contributors Listed Below - COPYRIGHT 2018,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
@@ -26,9 +26,11 @@
#include <devicefw/driverif.H>
#include <errl/errlentry.H>
#include <errl/errlmanager.H>
+#include <errl/errludtarget.H>
#include <targeting/common/predicates/predicates.H>
#include <targeting/common/utilFilter.H>
#include <targeting/common/targetservice.H>
+#include <arch/memorymap.H>
#include <arch/ppc.H>
#include "mmio.H"
@@ -43,6 +45,8 @@
trace_desc_t* g_trac_mmio = NULL;
TRAC_INIT(&g_trac_mmio, MMIO_COMP_NAME, 2*KILOBYTE, TRACE::BUFFER_SLOW);
+#define OMI_PER_MC 8
+
namespace MMIO
{
@@ -66,128 +70,118 @@ errlHndl_t mmioSetup()
errlHndl_t l_err = nullptr;
TRACFCOMP(g_trac_mmio, ENTER_MRK"mmioSetup");
- // called from istep 12.3
-
+ // called after OMI bars have been written to HW registers
do
{
- // Get the base BAR address for an OCMB and use it to calculate the base
- // BAR address for OCMB0 on PROC0 (beginning of reserved physical memory
- // for all OCMBs).
- // Each pair of OCMBs uses 8GB of interleaved memory,
- // the second OCMB's memory starts 2GB after the first's.
- TARGETING::TargetHandleList l_omiTargetList;
-
- getAllChiplets(l_omiTargetList, TARGETING::TYPE_OMI);
- if (l_omiTargetList.size() == 0)
- {
- TRACFCOMP(g_trac_mmio,
- INFO_MRK"mmioSetup: Exiting, non-OMI system");
- break;
- }
+ // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB
+ //
+ // loop through all the Memory Channels (MC Targets)
+ // call allocate of 32 GB virtual memory space with mmio_dev_map() for each MC
+ TARGETING::TargetHandleList l_mcTargetList;
+ getAllChiplets(l_mcTargetList, TARGETING::TYPE_MC);
- auto l_omi = l_omiTargetList[0];
- auto l_omiParentProc = getParentProc(l_omi);
- if (l_omiParentProc == nullptr)
+ for (auto & l_mcTarget: l_mcTargetList)
{
- TRACFCOMP(g_trac_mmio, ERR_MRK
- "mmioSetup: Unable to find the parent processor for an"
- " OMI(0x%X).",
- l_omi->getAttr<TARGETING::ATTR_HUID>());
- /*@
- * @errortype
- * @moduleid MMIO::MOD_MMIO_SETUP
- * @reasoncode MMIO::RC_PROC_NOT_FOUND
- * @userdata1 Target huid
- * @userdata2 None
- * @devdesc mmioSetup> Unable to find parent processor for OMI.
- * @custdesc Unexpected memory subsystem firmware error.
- */
- l_err = new ERRORLOG::ErrlEntry(
- ERRORLOG::ERRL_SEV_UNRECOVERABLE,
- MMIO::MOD_MMIO_SETUP,
- MMIO::RC_PROC_NOT_FOUND,
- l_omi->getAttr<TARGETING::ATTR_HUID>(),
- 0,
- ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
-
- break;
- }
-
- // There's a 1:1 relationship between OMIs and OCMBs, so we can directly
- // relate the OMI position and BAR base addr to its associated OCMB.
-
- // Get the position of the random OCMB. (OCMBs 0-15 will be on proc0,
- // 16-31 on proc1, etc)
- auto l_ocmbPos = l_omi->getAttr<TARGETING::ATTR_CHIP_UNIT>();
- l_ocmbPos += l_omiParentProc->getAttr<TARGETING::ATTR_POSITION>() *
- fapi2::MAX_OMI_PER_PROC;
+ uint32_t l_mcChipUnit =
+ l_mcTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>();
- // Get the base BAR address of the OCMB.
- auto l_ocmbBaseAddr =
- l_omi->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();
+ // Get the base BAR address for OpenCapi Memory Interfaces (OMIs) of the this Memory Channel (MC)
+ auto l_omiBaseAddr =
+ l_mcTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();
- // Calculate the base BAR address of OCMB0 on PROC0 by subtracting 8GB
- // for every pair of OCMBs beyond the first pair, and subtract an
- // additional 2GB if the initial OCMB is the second in a pair.
- l_ocmbBaseAddr -= ((l_ocmbPos / 2) * 8 * GIGABYTE) +
- ((l_ocmbPos % 2) * 2 * GIGABYTE);
+ // Apply the MMIO base offset so we get the real address
+ uint64_t l_realAddr = ( l_omiBaseAddr | MMIO_BASE );
- // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB
- //
- // loop through all the procs
- // call mmio_dev_map() on OCMBs 0-7 and 8-15
- // set VM_ADDR on each OCMB
- // each pair of OCMBs has their memories interleaved with their
- // 2GB config sections together and their 2GB mmio sections
- // together, we will be setting VM_ADDR to point to the cfg
- // section of each ocmb
- // Example
- // 0GB ocmb0 cfg
- // 2GB ocmb1 cfg
- // 4GB ocmb0 mmio
- // 6GB ocmb1 mmio
- TARGETING::TargetHandleList l_procTargetList;
-
- getAllChips(l_procTargetList, TARGETING::TYPE_PROC);
- for (auto & l_procTarget: l_procTargetList)
- {
- // map all 16 OCMBs, 8 OCMBs (32GB) at a time
- uint64_t *l_virtAddr[2] = {nullptr};
- uint32_t l_procNum =
- l_procTarget->getAttr<TARGETING::ATTR_POSITION>();
- uint64_t l_realAddr =
- l_ocmbBaseAddr + (l_procNum * 2 * THIRTYTWO_GB);
-
- l_virtAddr[0] = static_cast<uint64_t *>
- (mmio_dev_map(reinterpret_cast<void *>(l_realAddr),
- THIRTYTWO_GB));
- l_realAddr += THIRTYTWO_GB;
- l_virtAddr[1] = static_cast<uint64_t *>
+ // Map the device with a kernal call, each device, the MC, is 32 GB
+ uint64_t l_virtAddr = reinterpret_cast<uint64_t>
(mmio_dev_map(reinterpret_cast<void *>(l_realAddr),
THIRTYTWO_GB));
+ TRACFCOMP ( g_trac_mmio, "MC%.02x (0x%.08X) MMIO BAR PHYSICAL ADDR = 0x%lx VIRTUAL ADDR = 0x%lx" ,
+ l_mcChipUnit ? 0x23 : 0x01, TARGETING::get_huid(l_mcTarget),
+ l_realAddr, l_virtAddr);
+
// set VM_ADDR on each OCMB
- TARGETING::TargetHandleList l_ocmbTargetList;
- l_ocmbTargetList.clear();
- getChildAffinityTargets(l_ocmbTargetList, l_procTarget,
- TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP);
- for (auto & l_ocmbTarget: l_ocmbTargetList)
+ TARGETING::TargetHandleList l_omiTargetList;
+ getChildChiplets(l_omiTargetList, l_mcTarget, TARGETING::TYPE_OMI);
+
+ for (auto & l_omiTarget: l_omiTargetList)
{
- uint64_t l_ocmbVmAddr = 0;
- uint32_t l_ocmbNum =
- l_ocmbTarget->getAttr<TARGETING::ATTR_POSITION>();
+ // ATTR_CHIP_UNIT is relative to other OMI under this PROC
+ uint32_t l_omiChipUnit =
+ l_omiTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>();
+
+ // Get the OMI position relative to other OMIs under its parent MC chiplet
+ uint32_t l_omiPosRelativeToMc = l_omiChipUnit % OMI_PER_MC;
+
+ // Calculate what we think the real address for this OCMB should be. This should
+ // match what the ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET attribute is set to.
+
+ // Each Memory Controller Channel (MCC) uses 8 GB of Memory Mapped IO, 4 GB for each of its child OCMBs.
+ // Each OCMB has 2 MMIO distinct spaces that get mapped. The CONFIG space, and the MMIO space. The CONFIG
+ // Space is always before the MMIO space we will treat that as the BAR for the OCMB target. These
+ // paired OCMB spaces get interleaved as follows :
+ // ocmb | BAR ATTRIBUTE | Type | Base reg - end addr | size | sub-ch
+ // +-----+--------------------+------+-----------------------------------------+------+-------
+ // ocmb0 | 0x0006030200000000 | cnfg | 0x0006030200000000 - 0x000603027FFFFFFF | 2GB | 0
+ // ocmb1 | 0x0006030280000000 | cnfg | 0x0006030280000000 - 0x00060302FFFFFFFF | 2GB | 1
+ // ocmb0 | N/A | mmio | 0x0006030300000000 - 0x000603037FFFFFFF | 2GB | 0
+ // ocmb1 | N/A | mmio | 0x0006030380000000 - 0x00060303FFFFFFFF | 2GB | 1
+ // +-----+--------------------+------+-----------------------------------------+------+-------
+
+ // Calculate CNFG space BAR to write to OCMB attribute
+ uint64_t l_currentOmiOffset = (( l_omiPosRelativeToMc / 2) * 8 * GIGABYTE) +
+ (( l_omiPosRelativeToMc % 2) * 2 * GIGABYTE);
+
+ // Calculated real address for this OMI is (BAR from MC attribute) + (currentOmiOffset)
+ uint64_t l_calulatedRealAddr = l_omiBaseAddr + l_currentOmiOffset;
+
+ // Grab bar value from attribute to verify it matches our calculations
+ auto l_omiBarAttrVal = l_omiTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();
+
+ if(l_omiBarAttrVal != l_calulatedRealAddr)
+ {
+ TRACFCOMP(g_trac_mmio,
+ "Discrepancy found between calculated OMI MMIO bar offset and what we found in ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET");
+ TRACFCOMP(g_trac_mmio, "Calculated Offset: 0x%lx, Attribute Value : 0x%lx", l_calulatedRealAddr, l_omiBarAttrVal);
+
+ /*@
+ * @errortype ERRORLOG::ERRL_SEV_UNRECOVERABLE
+ * @moduleid MMIO::MOD_MMIO_SETUP
+ * @reasoncode MMIO::RC_BAR_OFFSET_MISMATCH
+ * @userdata1 Calculated Bar Offset
+ * @userdata2 Bar offset from attribute
+ * @devdesc mmioSetup> Mismatch between calculated map value
+ * and what is in attribute xml
+ * @custdesc Unexpected memory subsystem firmware error.
+ */
+ l_err = new ERRORLOG::ErrlEntry(
+ ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+ MMIO::MOD_MMIO_SETUP,
+ MMIO::RC_BAR_OFFSET_MISMATCH,
+ l_calulatedRealAddr,
+ l_omiBarAttrVal,
+ ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
+ l_err->collectTrace( MMIO_COMP_NAME);
+ ERRORLOG::ErrlUserDetailsTarget(l_omiTarget).addToLog(l_err);
+
+ break;
+ }
- // OCMBs 0-7 in first map, 8-15 in second map
- l_ocmbVmAddr =
- reinterpret_cast<uint64_t>(l_virtAddr[l_ocmbNum / 8]);
- // Each pair of OCMBs uses 8GB of interleaved memory,
- // the second OCMB's memory starts 2GB after the first's
- l_ocmbVmAddr += ((l_ocmbNum / 2) * 8 * GIGABYTE) +
- ((l_ocmbNum % 2) * 2 * GIGABYTE);
+ uint64_t l_currentOmiVirtAddr = l_virtAddr + l_currentOmiOffset;
- l_ocmbTarget->
- setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_ocmbVmAddr);
+ // set VM_ADDR the associated OCMB
+ TARGETING::TargetHandleList l_ocmbTargetList;
+ getChildAffinityTargets(l_ocmbTargetList, l_omiTarget,
+ TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP);
+
+ assert(l_ocmbTargetList.size() == 1 , "OCMB chips list found for a given OMI != 1 as expected");
+
+ TRACFCOMP(g_trac_mmio, "Setting HUID 0x%.08X MMIO vm addr to be 0x%lx , real address is 0x%lx", TARGETING::get_huid(l_ocmbTargetList[0]),
+ l_currentOmiVirtAddr, l_calulatedRealAddr | MMIO_BASE );
+
+ l_ocmbTargetList[0]->setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_currentOmiVirtAddr);
}
}
} while(0);
@@ -216,7 +210,7 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType,
TRACDCOMP(g_trac_mmio, ENTER_MRK"ocmbMmioPerformOp");
TRACDCOMP(g_trac_mmio, INFO_MRK"op=%d, target=0x%.8X",
- i_opType, i_target);
+ i_opType, TARGETING::get_huid(i_target));
TRACDCOMP(g_trac_mmio, INFO_MRK"buffer=%p, length=%d, accessType=%ld",
io_buffer, io_buflen, i_accessType);
TRACDCOMP(g_trac_mmio, INFO_MRK"offset=0x%lX, accessLimit=%ld",
@@ -226,6 +220,8 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType,
{
uint64_t l_addr = i_target->getAttr<TARGETING::ATTR_MMIO_VM_ADDR>();
+ TRACDCOMP(g_trac_mmio, INFO_MRK"MMIO Op l_addr=0x%lX ", l_addr);
+
if (l_addr == 0)
{
TRACFCOMP(g_trac_mmio, ERR_MRK
diff --git a/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml b/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml
index 7a14636cb..661da5a56 100644
--- a/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml
+++ b/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml
@@ -5,7 +5,7 @@
<!-- -->
<!-- OpenPOWER HostBoot Project -->
<!-- -->
-<!-- Contributors Listed Below - COPYRIGHT 2018 -->
+<!-- Contributors Listed Below - COPYRIGHT 2018,2019 -->
<!-- [+] International Business Machines Corp. -->
<!-- -->
<!-- -->
@@ -6512,6 +6512,11 @@
<id>TYPE</id>
<default>MC</default>
</attribute>
+ <attribute>
+ <id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id>
+ <!-- 3TB + 16GB -->
+ <default>0x30400000000</default>
+ </attribute>
</targetInstance>
<targetInstance>
@@ -6574,6 +6579,11 @@
<id>TYPE</id>
<default>MC</default>
</attribute>
+ <attribute>
+ <id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id>
+ <!-- 3TB + 48GB -->
+ <default>0x30C00000000</default>
+ </attribute>
</targetInstance>
<!-- ===================================================================== -->
OpenPOWER on IntegriCloud