diff options
Diffstat (limited to 'src/usr/mmio/mmio.C')
-rw-r--r-- | src/usr/mmio/mmio.C | 214 |
1 files changed, 105 insertions, 109 deletions
diff --git a/src/usr/mmio/mmio.C b/src/usr/mmio/mmio.C index cba75bb7c..1b84660f8 100644 --- a/src/usr/mmio/mmio.C +++ b/src/usr/mmio/mmio.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2018 */ +/* Contributors Listed Below - COPYRIGHT 2018,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -26,9 +26,11 @@ #include <devicefw/driverif.H> #include <errl/errlentry.H> #include <errl/errlmanager.H> +#include <errl/errludtarget.H> #include <targeting/common/predicates/predicates.H> #include <targeting/common/utilFilter.H> #include <targeting/common/targetservice.H> +#include <arch/memorymap.H> #include <arch/ppc.H> #include "mmio.H" @@ -43,6 +45,8 @@ trace_desc_t* g_trac_mmio = NULL; TRAC_INIT(&g_trac_mmio, MMIO_COMP_NAME, 2*KILOBYTE, TRACE::BUFFER_SLOW); +#define OMI_PER_MC 8 + namespace MMIO { @@ -66,128 +70,118 @@ errlHndl_t mmioSetup() errlHndl_t l_err = nullptr; TRACFCOMP(g_trac_mmio, ENTER_MRK"mmioSetup"); - // called from istep 12.3 - + // called after OMI bars have been written to HW registers do { - // Get the base BAR address for an OCMB and use it to calculate the base - // BAR address for OCMB0 on PROC0 (beginning of reserved physical memory - // for all OCMBs). - // Each pair of OCMBs uses 8GB of interleaved memory, - // the second OCMB's memory starts 2GB after the first's. - TARGETING::TargetHandleList l_omiTargetList; - - getAllChiplets(l_omiTargetList, TARGETING::TYPE_OMI); - if (l_omiTargetList.size() == 0) - { - TRACFCOMP(g_trac_mmio, - INFO_MRK"mmioSetup: Exiting, non-OMI system"); - break; - } + // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB + // + // loop through all the Memory Channels (MC Targets) + // call allocate of 32 GB virtual memory space with mmio_dev_map() for each MC + TARGETING::TargetHandleList l_mcTargetList; + getAllChiplets(l_mcTargetList, TARGETING::TYPE_MC); - auto l_omi = l_omiTargetList[0]; - auto l_omiParentProc = getParentProc(l_omi); - if (l_omiParentProc == nullptr) + for (auto & l_mcTarget: l_mcTargetList) { - TRACFCOMP(g_trac_mmio, ERR_MRK - "mmioSetup: Unable to find the parent processor for an" - " OMI(0x%X).", - l_omi->getAttr<TARGETING::ATTR_HUID>()); - /*@ - * @errortype - * @moduleid MMIO::MOD_MMIO_SETUP - * @reasoncode MMIO::RC_PROC_NOT_FOUND - * @userdata1 Target huid - * @userdata2 None - * @devdesc mmioSetup> Unable to find parent processor for OMI. - * @custdesc Unexpected memory subsystem firmware error. - */ - l_err = new ERRORLOG::ErrlEntry( - ERRORLOG::ERRL_SEV_UNRECOVERABLE, - MMIO::MOD_MMIO_SETUP, - MMIO::RC_PROC_NOT_FOUND, - l_omi->getAttr<TARGETING::ATTR_HUID>(), - 0, - ERRORLOG::ErrlEntry::ADD_SW_CALLOUT); - - break; - } - - // There's a 1:1 relationship between OMIs and OCMBs, so we can directly - // relate the OMI position and BAR base addr to its associated OCMB. - - // Get the position of the random OCMB. (OCMBs 0-15 will be on proc0, - // 16-31 on proc1, etc) - auto l_ocmbPos = l_omi->getAttr<TARGETING::ATTR_CHIP_UNIT>(); - l_ocmbPos += l_omiParentProc->getAttr<TARGETING::ATTR_POSITION>() * - fapi2::MAX_OMI_PER_PROC; + uint32_t l_mcChipUnit = + l_mcTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>(); - // Get the base BAR address of the OCMB. - auto l_ocmbBaseAddr = - l_omi->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); + // Get the base BAR address for OpenCapi Memory Interfaces (OMIs) of the this Memory Channel (MC) + auto l_omiBaseAddr = + l_mcTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); - // Calculate the base BAR address of OCMB0 on PROC0 by subtracting 8GB - // for every pair of OCMBs beyond the first pair, and subtract an - // additional 2GB if the initial OCMB is the second in a pair. - l_ocmbBaseAddr -= ((l_ocmbPos / 2) * 8 * GIGABYTE) + - ((l_ocmbPos % 2) * 2 * GIGABYTE); + // Apply the MMIO base offset so we get the real address + uint64_t l_realAddr = ( l_omiBaseAddr | MMIO_BASE ); - // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB - // - // loop through all the procs - // call mmio_dev_map() on OCMBs 0-7 and 8-15 - // set VM_ADDR on each OCMB - // each pair of OCMBs has their memories interleaved with their - // 2GB config sections together and their 2GB mmio sections - // together, we will be setting VM_ADDR to point to the cfg - // section of each ocmb - // Example - // 0GB ocmb0 cfg - // 2GB ocmb1 cfg - // 4GB ocmb0 mmio - // 6GB ocmb1 mmio - TARGETING::TargetHandleList l_procTargetList; - - getAllChips(l_procTargetList, TARGETING::TYPE_PROC); - for (auto & l_procTarget: l_procTargetList) - { - // map all 16 OCMBs, 8 OCMBs (32GB) at a time - uint64_t *l_virtAddr[2] = {nullptr}; - uint32_t l_procNum = - l_procTarget->getAttr<TARGETING::ATTR_POSITION>(); - uint64_t l_realAddr = - l_ocmbBaseAddr + (l_procNum * 2 * THIRTYTWO_GB); - - l_virtAddr[0] = static_cast<uint64_t *> - (mmio_dev_map(reinterpret_cast<void *>(l_realAddr), - THIRTYTWO_GB)); - l_realAddr += THIRTYTWO_GB; - l_virtAddr[1] = static_cast<uint64_t *> + // Map the device with a kernal call, each device, the MC, is 32 GB + uint64_t l_virtAddr = reinterpret_cast<uint64_t> (mmio_dev_map(reinterpret_cast<void *>(l_realAddr), THIRTYTWO_GB)); + TRACFCOMP ( g_trac_mmio, "MC%.02x (0x%.08X) MMIO BAR PHYSICAL ADDR = 0x%lx VIRTUAL ADDR = 0x%lx" , + l_mcChipUnit ? 0x23 : 0x01, TARGETING::get_huid(l_mcTarget), + l_realAddr, l_virtAddr); + // set VM_ADDR on each OCMB - TARGETING::TargetHandleList l_ocmbTargetList; - l_ocmbTargetList.clear(); - getChildAffinityTargets(l_ocmbTargetList, l_procTarget, - TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP); - for (auto & l_ocmbTarget: l_ocmbTargetList) + TARGETING::TargetHandleList l_omiTargetList; + getChildChiplets(l_omiTargetList, l_mcTarget, TARGETING::TYPE_OMI); + + for (auto & l_omiTarget: l_omiTargetList) { - uint64_t l_ocmbVmAddr = 0; - uint32_t l_ocmbNum = - l_ocmbTarget->getAttr<TARGETING::ATTR_POSITION>(); + // ATTR_CHIP_UNIT is relative to other OMI under this PROC + uint32_t l_omiChipUnit = + l_omiTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>(); + + // Get the OMI position relative to other OMIs under its parent MC chiplet + uint32_t l_omiPosRelativeToMc = l_omiChipUnit % OMI_PER_MC; + + // Calculate what we think the real address for this OCMB should be. This should + // match what the ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET attribute is set to. + + // Each Memory Controller Channel (MCC) uses 8 GB of Memory Mapped IO, 4 GB for each of its child OCMBs. + // Each OCMB has 2 MMIO distinct spaces that get mapped. The CONFIG space, and the MMIO space. The CONFIG + // Space is always before the MMIO space we will treat that as the BAR for the OCMB target. These + // paired OCMB spaces get interleaved as follows : + // ocmb | BAR ATTRIBUTE | Type | Base reg - end addr | size | sub-ch + // +-----+--------------------+------+-----------------------------------------+------+------- + // ocmb0 | 0x0006030200000000 | cnfg | 0x0006030200000000 - 0x000603027FFFFFFF | 2GB | 0 + // ocmb1 | 0x0006030280000000 | cnfg | 0x0006030280000000 - 0x00060302FFFFFFFF | 2GB | 1 + // ocmb0 | N/A | mmio | 0x0006030300000000 - 0x000603037FFFFFFF | 2GB | 0 + // ocmb1 | N/A | mmio | 0x0006030380000000 - 0x00060303FFFFFFFF | 2GB | 1 + // +-----+--------------------+------+-----------------------------------------+------+------- + + // Calculate CNFG space BAR to write to OCMB attribute + uint64_t l_currentOmiOffset = (( l_omiPosRelativeToMc / 2) * 8 * GIGABYTE) + + (( l_omiPosRelativeToMc % 2) * 2 * GIGABYTE); + + // Calculated real address for this OMI is (BAR from MC attribute) + (currentOmiOffset) + uint64_t l_calulatedRealAddr = l_omiBaseAddr + l_currentOmiOffset; + + // Grab bar value from attribute to verify it matches our calculations + auto l_omiBarAttrVal = l_omiTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); + + if(l_omiBarAttrVal != l_calulatedRealAddr) + { + TRACFCOMP(g_trac_mmio, + "Discrepancy found between calculated OMI MMIO bar offset and what we found in ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET"); + TRACFCOMP(g_trac_mmio, "Calculated Offset: 0x%lx, Attribute Value : 0x%lx", l_calulatedRealAddr, l_omiBarAttrVal); + + /*@ + * @errortype ERRORLOG::ERRL_SEV_UNRECOVERABLE + * @moduleid MMIO::MOD_MMIO_SETUP + * @reasoncode MMIO::RC_BAR_OFFSET_MISMATCH + * @userdata1 Calculated Bar Offset + * @userdata2 Bar offset from attribute + * @devdesc mmioSetup> Mismatch between calculated map value + * and what is in attribute xml + * @custdesc Unexpected memory subsystem firmware error. + */ + l_err = new ERRORLOG::ErrlEntry( + ERRORLOG::ERRL_SEV_UNRECOVERABLE, + MMIO::MOD_MMIO_SETUP, + MMIO::RC_BAR_OFFSET_MISMATCH, + l_calulatedRealAddr, + l_omiBarAttrVal, + ERRORLOG::ErrlEntry::ADD_SW_CALLOUT); + l_err->collectTrace( MMIO_COMP_NAME); + ERRORLOG::ErrlUserDetailsTarget(l_omiTarget).addToLog(l_err); + + break; + } - // OCMBs 0-7 in first map, 8-15 in second map - l_ocmbVmAddr = - reinterpret_cast<uint64_t>(l_virtAddr[l_ocmbNum / 8]); - // Each pair of OCMBs uses 8GB of interleaved memory, - // the second OCMB's memory starts 2GB after the first's - l_ocmbVmAddr += ((l_ocmbNum / 2) * 8 * GIGABYTE) + - ((l_ocmbNum % 2) * 2 * GIGABYTE); + uint64_t l_currentOmiVirtAddr = l_virtAddr + l_currentOmiOffset; - l_ocmbTarget-> - setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_ocmbVmAddr); + // set VM_ADDR the associated OCMB + TARGETING::TargetHandleList l_ocmbTargetList; + getChildAffinityTargets(l_ocmbTargetList, l_omiTarget, + TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP); + + assert(l_ocmbTargetList.size() == 1 , "OCMB chips list found for a given OMI != 1 as expected"); + + TRACFCOMP(g_trac_mmio, "Setting HUID 0x%.08X MMIO vm addr to be 0x%lx , real address is 0x%lx", TARGETING::get_huid(l_ocmbTargetList[0]), + l_currentOmiVirtAddr, l_calulatedRealAddr | MMIO_BASE ); + + l_ocmbTargetList[0]->setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_currentOmiVirtAddr); } } } while(0); @@ -216,7 +210,7 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType, TRACDCOMP(g_trac_mmio, ENTER_MRK"ocmbMmioPerformOp"); TRACDCOMP(g_trac_mmio, INFO_MRK"op=%d, target=0x%.8X", - i_opType, i_target); + i_opType, TARGETING::get_huid(i_target)); TRACDCOMP(g_trac_mmio, INFO_MRK"buffer=%p, length=%d, accessType=%ld", io_buffer, io_buflen, i_accessType); TRACDCOMP(g_trac_mmio, INFO_MRK"offset=0x%lX, accessLimit=%ld", @@ -226,6 +220,8 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType, { uint64_t l_addr = i_target->getAttr<TARGETING::ATTR_MMIO_VM_ADDR>(); + TRACDCOMP(g_trac_mmio, INFO_MRK"MMIO Op l_addr=0x%lX ", l_addr); + if (l_addr == 0) { TRACFCOMP(g_trac_mmio, ERR_MRK |