diff options
-rw-r--r-- | src/include/arch/memorymap.H | 3 | ||||
-rw-r--r-- | src/include/usr/mmio/mmio_reasoncodes.H | 3 | ||||
-rw-r--r-- | src/usr/ibscom/ibscom.C | 5 | ||||
-rw-r--r-- | src/usr/mmio/mmio.C | 214 | ||||
-rw-r--r-- | src/usr/targeting/common/xmltohb/simics_AXONE.system.xml | 12 |
5 files changed, 122 insertions, 115 deletions
diff --git a/src/include/arch/memorymap.H b/src/include/arch/memorymap.H index a25bcb62d..7b75f6363 100644 --- a/src/include/arch/memorymap.H +++ b/src/include/arch/memorymap.H @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2017,2018 */ +/* Contributors Listed Below - COPYRIGHT 2017,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -38,6 +38,7 @@ constexpr uint64_t MMIO_OFFSET_PER_CHIP = (4*TERABYTE); //0x40000000000 constexpr uint64_t MMIO_OFFSET_PER_GROUP = (32*TERABYTE); //0x200000000000 +constexpr uint64_t MMIO_BASE = 0x6000000000000; /** * @brief Compute MMIO value for a given chip and base value */ diff --git a/src/include/usr/mmio/mmio_reasoncodes.H b/src/include/usr/mmio/mmio_reasoncodes.H index 7e2481934..86ff60b5b 100644 --- a/src/include/usr/mmio/mmio_reasoncodes.H +++ b/src/include/usr/mmio/mmio_reasoncodes.H @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2011,2018 */ +/* Contributors Listed Below - COPYRIGHT 2011,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -51,6 +51,7 @@ namespace MMIO RC_BAD_MMIO_READ = MMIO_COMP_ID | 0x08, RC_BAD_MMIO_WRITE = MMIO_COMP_ID | 0x09, RC_PROC_NOT_FOUND = MMIO_COMP_ID | 0x0A, + RC_BAR_OFFSET_MISMATCH = MMIO_COMP_ID | 0x0B, }; }; diff --git a/src/usr/ibscom/ibscom.C b/src/usr/ibscom/ibscom.C index e3cb6d468..8939454a3 100644 --- a/src/usr/ibscom/ibscom.C +++ b/src/usr/ibscom/ibscom.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2012,2018 */ +/* Contributors Listed Below - COPYRIGHT 2012,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -66,7 +66,6 @@ namespace IBSCOM // SCOM Register addresses const uint32_t MBS_FIR = 0x02011400; const uint32_t MBSIBERR0 = 0x0201141B; -const uint64_t IBSCOM_BASE = 0x0006000000000000; const uint64_t BIT_18_MASK = 0x0000000000002000; // Register XSCcom access functions to DD framework @@ -217,7 +216,7 @@ errlHndl_t getTargetVirtualAddress(Target* i_target, uint8_t l_chipId = l_parentChip->getAttr<ATTR_PROC_EFF_FABRIC_CHIP_ID>(); - l_IBScomAddr = computeMemoryMapOffset( IBSCOM_BASE, + l_IBScomAddr = computeMemoryMapOffset( MMIO_BASE, l_groupId, l_chipId ); l_IBScomAddr = l_IBScomAddr + l_IBScomBaseAddrOffset; diff --git a/src/usr/mmio/mmio.C b/src/usr/mmio/mmio.C index cba75bb7c..1b84660f8 100644 --- a/src/usr/mmio/mmio.C +++ b/src/usr/mmio/mmio.C @@ -5,7 +5,7 @@ /* */ /* OpenPOWER HostBoot Project */ /* */ -/* Contributors Listed Below - COPYRIGHT 2018 */ +/* Contributors Listed Below - COPYRIGHT 2018,2019 */ /* [+] International Business Machines Corp. */ /* */ /* */ @@ -26,9 +26,11 @@ #include <devicefw/driverif.H> #include <errl/errlentry.H> #include <errl/errlmanager.H> +#include <errl/errludtarget.H> #include <targeting/common/predicates/predicates.H> #include <targeting/common/utilFilter.H> #include <targeting/common/targetservice.H> +#include <arch/memorymap.H> #include <arch/ppc.H> #include "mmio.H" @@ -43,6 +45,8 @@ trace_desc_t* g_trac_mmio = NULL; TRAC_INIT(&g_trac_mmio, MMIO_COMP_NAME, 2*KILOBYTE, TRACE::BUFFER_SLOW); +#define OMI_PER_MC 8 + namespace MMIO { @@ -66,128 +70,118 @@ errlHndl_t mmioSetup() errlHndl_t l_err = nullptr; TRACFCOMP(g_trac_mmio, ENTER_MRK"mmioSetup"); - // called from istep 12.3 - + // called after OMI bars have been written to HW registers do { - // Get the base BAR address for an OCMB and use it to calculate the base - // BAR address for OCMB0 on PROC0 (beginning of reserved physical memory - // for all OCMBs). - // Each pair of OCMBs uses 8GB of interleaved memory, - // the second OCMB's memory starts 2GB after the first's. - TARGETING::TargetHandleList l_omiTargetList; - - getAllChiplets(l_omiTargetList, TARGETING::TYPE_OMI); - if (l_omiTargetList.size() == 0) - { - TRACFCOMP(g_trac_mmio, - INFO_MRK"mmioSetup: Exiting, non-OMI system"); - break; - } + // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB + // + // loop through all the Memory Channels (MC Targets) + // call allocate of 32 GB virtual memory space with mmio_dev_map() for each MC + TARGETING::TargetHandleList l_mcTargetList; + getAllChiplets(l_mcTargetList, TARGETING::TYPE_MC); - auto l_omi = l_omiTargetList[0]; - auto l_omiParentProc = getParentProc(l_omi); - if (l_omiParentProc == nullptr) + for (auto & l_mcTarget: l_mcTargetList) { - TRACFCOMP(g_trac_mmio, ERR_MRK - "mmioSetup: Unable to find the parent processor for an" - " OMI(0x%X).", - l_omi->getAttr<TARGETING::ATTR_HUID>()); - /*@ - * @errortype - * @moduleid MMIO::MOD_MMIO_SETUP - * @reasoncode MMIO::RC_PROC_NOT_FOUND - * @userdata1 Target huid - * @userdata2 None - * @devdesc mmioSetup> Unable to find parent processor for OMI. - * @custdesc Unexpected memory subsystem firmware error. - */ - l_err = new ERRORLOG::ErrlEntry( - ERRORLOG::ERRL_SEV_UNRECOVERABLE, - MMIO::MOD_MMIO_SETUP, - MMIO::RC_PROC_NOT_FOUND, - l_omi->getAttr<TARGETING::ATTR_HUID>(), - 0, - ERRORLOG::ErrlEntry::ADD_SW_CALLOUT); - - break; - } - - // There's a 1:1 relationship between OMIs and OCMBs, so we can directly - // relate the OMI position and BAR base addr to its associated OCMB. - - // Get the position of the random OCMB. (OCMBs 0-15 will be on proc0, - // 16-31 on proc1, etc) - auto l_ocmbPos = l_omi->getAttr<TARGETING::ATTR_CHIP_UNIT>(); - l_ocmbPos += l_omiParentProc->getAttr<TARGETING::ATTR_POSITION>() * - fapi2::MAX_OMI_PER_PROC; + uint32_t l_mcChipUnit = + l_mcTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>(); - // Get the base BAR address of the OCMB. - auto l_ocmbBaseAddr = - l_omi->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); + // Get the base BAR address for OpenCapi Memory Interfaces (OMIs) of the this Memory Channel (MC) + auto l_omiBaseAddr = + l_mcTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); - // Calculate the base BAR address of OCMB0 on PROC0 by subtracting 8GB - // for every pair of OCMBs beyond the first pair, and subtract an - // additional 2GB if the initial OCMB is the second in a pair. - l_ocmbBaseAddr -= ((l_ocmbPos / 2) * 8 * GIGABYTE) + - ((l_ocmbPos % 2) * 2 * GIGABYTE); + // Apply the MMIO base offset so we get the real address + uint64_t l_realAddr = ( l_omiBaseAddr | MMIO_BASE ); - // map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB - // - // loop through all the procs - // call mmio_dev_map() on OCMBs 0-7 and 8-15 - // set VM_ADDR on each OCMB - // each pair of OCMBs has their memories interleaved with their - // 2GB config sections together and their 2GB mmio sections - // together, we will be setting VM_ADDR to point to the cfg - // section of each ocmb - // Example - // 0GB ocmb0 cfg - // 2GB ocmb1 cfg - // 4GB ocmb0 mmio - // 6GB ocmb1 mmio - TARGETING::TargetHandleList l_procTargetList; - - getAllChips(l_procTargetList, TARGETING::TYPE_PROC); - for (auto & l_procTarget: l_procTargetList) - { - // map all 16 OCMBs, 8 OCMBs (32GB) at a time - uint64_t *l_virtAddr[2] = {nullptr}; - uint32_t l_procNum = - l_procTarget->getAttr<TARGETING::ATTR_POSITION>(); - uint64_t l_realAddr = - l_ocmbBaseAddr + (l_procNum * 2 * THIRTYTWO_GB); - - l_virtAddr[0] = static_cast<uint64_t *> - (mmio_dev_map(reinterpret_cast<void *>(l_realAddr), - THIRTYTWO_GB)); - l_realAddr += THIRTYTWO_GB; - l_virtAddr[1] = static_cast<uint64_t *> + // Map the device with a kernal call, each device, the MC, is 32 GB + uint64_t l_virtAddr = reinterpret_cast<uint64_t> (mmio_dev_map(reinterpret_cast<void *>(l_realAddr), THIRTYTWO_GB)); + TRACFCOMP ( g_trac_mmio, "MC%.02x (0x%.08X) MMIO BAR PHYSICAL ADDR = 0x%lx VIRTUAL ADDR = 0x%lx" , + l_mcChipUnit ? 0x23 : 0x01, TARGETING::get_huid(l_mcTarget), + l_realAddr, l_virtAddr); + // set VM_ADDR on each OCMB - TARGETING::TargetHandleList l_ocmbTargetList; - l_ocmbTargetList.clear(); - getChildAffinityTargets(l_ocmbTargetList, l_procTarget, - TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP); - for (auto & l_ocmbTarget: l_ocmbTargetList) + TARGETING::TargetHandleList l_omiTargetList; + getChildChiplets(l_omiTargetList, l_mcTarget, TARGETING::TYPE_OMI); + + for (auto & l_omiTarget: l_omiTargetList) { - uint64_t l_ocmbVmAddr = 0; - uint32_t l_ocmbNum = - l_ocmbTarget->getAttr<TARGETING::ATTR_POSITION>(); + // ATTR_CHIP_UNIT is relative to other OMI under this PROC + uint32_t l_omiChipUnit = + l_omiTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>(); + + // Get the OMI position relative to other OMIs under its parent MC chiplet + uint32_t l_omiPosRelativeToMc = l_omiChipUnit % OMI_PER_MC; + + // Calculate what we think the real address for this OCMB should be. This should + // match what the ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET attribute is set to. + + // Each Memory Controller Channel (MCC) uses 8 GB of Memory Mapped IO, 4 GB for each of its child OCMBs. + // Each OCMB has 2 MMIO distinct spaces that get mapped. The CONFIG space, and the MMIO space. The CONFIG + // Space is always before the MMIO space we will treat that as the BAR for the OCMB target. These + // paired OCMB spaces get interleaved as follows : + // ocmb | BAR ATTRIBUTE | Type | Base reg - end addr | size | sub-ch + // +-----+--------------------+------+-----------------------------------------+------+------- + // ocmb0 | 0x0006030200000000 | cnfg | 0x0006030200000000 - 0x000603027FFFFFFF | 2GB | 0 + // ocmb1 | 0x0006030280000000 | cnfg | 0x0006030280000000 - 0x00060302FFFFFFFF | 2GB | 1 + // ocmb0 | N/A | mmio | 0x0006030300000000 - 0x000603037FFFFFFF | 2GB | 0 + // ocmb1 | N/A | mmio | 0x0006030380000000 - 0x00060303FFFFFFFF | 2GB | 1 + // +-----+--------------------+------+-----------------------------------------+------+------- + + // Calculate CNFG space BAR to write to OCMB attribute + uint64_t l_currentOmiOffset = (( l_omiPosRelativeToMc / 2) * 8 * GIGABYTE) + + (( l_omiPosRelativeToMc % 2) * 2 * GIGABYTE); + + // Calculated real address for this OMI is (BAR from MC attribute) + (currentOmiOffset) + uint64_t l_calulatedRealAddr = l_omiBaseAddr + l_currentOmiOffset; + + // Grab bar value from attribute to verify it matches our calculations + auto l_omiBarAttrVal = l_omiTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>(); + + if(l_omiBarAttrVal != l_calulatedRealAddr) + { + TRACFCOMP(g_trac_mmio, + "Discrepancy found between calculated OMI MMIO bar offset and what we found in ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET"); + TRACFCOMP(g_trac_mmio, "Calculated Offset: 0x%lx, Attribute Value : 0x%lx", l_calulatedRealAddr, l_omiBarAttrVal); + + /*@ + * @errortype ERRORLOG::ERRL_SEV_UNRECOVERABLE + * @moduleid MMIO::MOD_MMIO_SETUP + * @reasoncode MMIO::RC_BAR_OFFSET_MISMATCH + * @userdata1 Calculated Bar Offset + * @userdata2 Bar offset from attribute + * @devdesc mmioSetup> Mismatch between calculated map value + * and what is in attribute xml + * @custdesc Unexpected memory subsystem firmware error. + */ + l_err = new ERRORLOG::ErrlEntry( + ERRORLOG::ERRL_SEV_UNRECOVERABLE, + MMIO::MOD_MMIO_SETUP, + MMIO::RC_BAR_OFFSET_MISMATCH, + l_calulatedRealAddr, + l_omiBarAttrVal, + ERRORLOG::ErrlEntry::ADD_SW_CALLOUT); + l_err->collectTrace( MMIO_COMP_NAME); + ERRORLOG::ErrlUserDetailsTarget(l_omiTarget).addToLog(l_err); + + break; + } - // OCMBs 0-7 in first map, 8-15 in second map - l_ocmbVmAddr = - reinterpret_cast<uint64_t>(l_virtAddr[l_ocmbNum / 8]); - // Each pair of OCMBs uses 8GB of interleaved memory, - // the second OCMB's memory starts 2GB after the first's - l_ocmbVmAddr += ((l_ocmbNum / 2) * 8 * GIGABYTE) + - ((l_ocmbNum % 2) * 2 * GIGABYTE); + uint64_t l_currentOmiVirtAddr = l_virtAddr + l_currentOmiOffset; - l_ocmbTarget-> - setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_ocmbVmAddr); + // set VM_ADDR the associated OCMB + TARGETING::TargetHandleList l_ocmbTargetList; + getChildAffinityTargets(l_ocmbTargetList, l_omiTarget, + TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP); + + assert(l_ocmbTargetList.size() == 1 , "OCMB chips list found for a given OMI != 1 as expected"); + + TRACFCOMP(g_trac_mmio, "Setting HUID 0x%.08X MMIO vm addr to be 0x%lx , real address is 0x%lx", TARGETING::get_huid(l_ocmbTargetList[0]), + l_currentOmiVirtAddr, l_calulatedRealAddr | MMIO_BASE ); + + l_ocmbTargetList[0]->setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_currentOmiVirtAddr); } } } while(0); @@ -216,7 +210,7 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType, TRACDCOMP(g_trac_mmio, ENTER_MRK"ocmbMmioPerformOp"); TRACDCOMP(g_trac_mmio, INFO_MRK"op=%d, target=0x%.8X", - i_opType, i_target); + i_opType, TARGETING::get_huid(i_target)); TRACDCOMP(g_trac_mmio, INFO_MRK"buffer=%p, length=%d, accessType=%ld", io_buffer, io_buflen, i_accessType); TRACDCOMP(g_trac_mmio, INFO_MRK"offset=0x%lX, accessLimit=%ld", @@ -226,6 +220,8 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType, { uint64_t l_addr = i_target->getAttr<TARGETING::ATTR_MMIO_VM_ADDR>(); + TRACDCOMP(g_trac_mmio, INFO_MRK"MMIO Op l_addr=0x%lX ", l_addr); + if (l_addr == 0) { TRACFCOMP(g_trac_mmio, ERR_MRK diff --git a/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml b/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml index 7a14636cb..661da5a56 100644 --- a/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml +++ b/src/usr/targeting/common/xmltohb/simics_AXONE.system.xml @@ -5,7 +5,7 @@ <!-- --> <!-- OpenPOWER HostBoot Project --> <!-- --> -<!-- Contributors Listed Below - COPYRIGHT 2018 --> +<!-- Contributors Listed Below - COPYRIGHT 2018,2019 --> <!-- [+] International Business Machines Corp. --> <!-- --> <!-- --> @@ -6512,6 +6512,11 @@ <id>TYPE</id> <default>MC</default> </attribute> + <attribute> + <id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id> + <!-- 3TB + 16GB --> + <default>0x30400000000</default> + </attribute> </targetInstance> <targetInstance> @@ -6574,6 +6579,11 @@ <id>TYPE</id> <default>MC</default> </attribute> + <attribute> + <id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id> + <!-- 3TB + 48GB --> + <default>0x30C00000000</default> + </attribute> </targetInstance> <!-- ===================================================================== --> |