Add Functionality To Allow FSP to Enable TCEs For Testing

This commit adds an attribute that the FSP team can use to tell
Hostboot to enable the TCE path for DMAs.  When the FSP sets this
attribute, the hostboot code will initialize the TCE settings.

Change-Id: I24c71f31264645cac5840caad1b59b90fe465fb4
RTC:168745
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/48036
Reviewed-by: Stephen M. Cprek <smcprek@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

9 files changed, 242 insertions, 89 deletions

diff --git a/src/include/usr/util/utiltce.H b/src/include/usr/util/utiltce.H
index bcaeecbca..f71946fee 100644
--- a/src/include/usr/util/utiltce.H
+++ b/src/include/usr/util/utiltce.H
@@ -106,6 +106,28 @@ errlHndl_t utilSetupPayloadTces(void);
  */
 errlHndl_t utilClosePayloadTces(void);
 
+/******************************************************/
+/* Miscellaneous Functions                            */
+/******************************************************/
+
+/**
+ * @brief Indicates if TCEs should be used for DMA path on the system
+ *
+ * @return bool - TRUE if TCEs should be used; False, if not
+ *
+ */
+bool utilUseTcesForDmas(void);
+
+/**
+ * @brief Enables the System to use TCEs without setting up a TCE Table.
+ *        This prevents the FSP from using unsecure DMAs during the IPL until
+ *        Hostboot sets up the TCE Table with proper TCEs.
+ *
+ * @return errlHndl_t - Return Error Handle if failed
+ *
+ */
+errlHndl_t utilEnableTcesWithoutTceTable(void);
+
 };
 
 #endif
diff --git a/src/usr/isteps/istep10/call_proc_chiplet_scominit.C b/src/usr/isteps/istep10/call_proc_chiplet_scominit.C
index f7df9989a..51fa4b75b 100644
--- a/src/usr/isteps/istep10/call_proc_chiplet_scominit.C
+++ b/src/usr/isteps/istep10/call_proc_chiplet_scominit.C
@@ -63,6 +63,9 @@
 #include <p9_chiplet_scominit.H>
 #include <p9_psi_scominit.H>
 
+// Util TCE Support
+#include <util/utiltce.H>
+
 namespace   ISTEP_10
 {
 
@@ -141,6 +144,30 @@ void*    call_proc_chiplet_scominit( void    *io_pArgs )
         }
     } // end of going through all processors
 
+    // Enable TCEs with an empty TCE Table, if necessary
+    // This will prevent the FSP from DMAing to system memory without
+    // hostboot's knowledge
+    if ( TCE::utilUseTcesForDmas() )
+    {
+        l_err = TCE::utilEnableTcesWithoutTceTable();
+
+        if (l_err)
+        {
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+             "call_proc_chiplet_scominit: utilEnableTcesWithoutTceTable "
+             "returned ERROR 0x%.4X",
+             l_err->reasonCode());
+
+            // Create IStep error log and cross ref to error that occurred
+            l_StepError.addErrorDetails( l_err );
+
+            // Commit the error log and move on
+            // Note: Error log should already be deleted and set to NULL
+            // after committing
+            errlCommit(l_err, HWPF_COMP_ID);
+        }
+    }
+
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                              "call_proc_chiplet_scominit exit" );
 
diff --git a/src/usr/isteps/istep16/call_host_ipl_complete.C b/src/usr/isteps/istep16/call_host_ipl_complete.C
index 87eca889b..12397d6c6 100644
--- a/src/usr/isteps/istep16/call_host_ipl_complete.C
+++ b/src/usr/isteps/istep16/call_host_ipl_complete.C
@@ -86,20 +86,19 @@ void* call_host_ipl_complete (void *io_pArgs)
         }
 
 
-
         // Setup the TCEs needed for the FSP to DMA the PAYLOAD
-/*      @TODO RTC 168745 - make this call when FSP is ready for TCE Support
- *                         and add check that we're on a FSP system
- *      NOTE:  add check to do this only on FSP-based systems
- *        l_err = TCE::utilSetupPayloadTces();
- *
- *       if( l_err )
- *       {
- *           TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
- *                      ERR_MRK"TCE::utilSetupPayloadTces failed");
- *           break;
- *       }
- */
+        if (TCE::utilUseTcesForDmas())
+        {
+            l_err = TCE::utilSetupPayloadTces();
+
+            if( l_err )
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           ERR_MRK"TCE::utilSetupPayloadTces failed");
+                // Don't continue with the rest of this istep
+                break;
+            }
+        }
 
         // Initialize the RUNTIME DATA attributes
         // that HDAT needs to allocate memory for us.
diff --git a/src/usr/isteps/istep21/call_host_runtime_setup.C b/src/usr/isteps/istep21/call_host_runtime_setup.C
index 48bda6b9b..767f7c422 100644
--- a/src/usr/isteps/istep21/call_host_runtime_setup.C
+++ b/src/usr/isteps/istep21/call_host_runtime_setup.C
@@ -70,6 +70,31 @@ void* call_host_runtime_setup (void *io_pArgs)
 
     do
     {
+        // Close PAYLOAD TCEs
+        // @TODO RTC 168745 - also close HDAT TCEs
+        if (TCE::utilUseTcesForDmas())
+        {
+
+            l_err = TCE::utilClosePayloadTces();
+            if ( l_err )
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           "Failed TCE::utilClosePayloadTces" );
+                // break from do loop if error occurred
+                break;
+            }
+
+            // Disable all TCEs
+            l_err = TCE::utilDisableTces();
+            if ( l_err )
+            {
+                TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
+                           "Failed TCE::utilDisableTces" );
+                // break from do loop if error occurred
+                break;
+            }
+        }
+
         //Need to send System Configuration down to SBE
         //Use targeting code to get a list of all processors
         TARGETING::TargetHandleList l_procChips;
@@ -340,32 +365,6 @@ void* call_host_runtime_setup (void *io_pArgs)
         }
 #endif
 
-        // Close PAYLOAD TCEs
-/*      @TODO RTC 168745 - make this call when FSP is ready for TCE Support
- *                         and add check that we're on a FSP system
- *      NOTE:  move this call to the start of this function
- *      NOTE:  add check to do this only on FSP-based systems
- *        l_err = TCE::utilClosePayloadTces();
- *       if ( l_err )
- *       {
- *           TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
- *                      "Failed TCE::utilClosePayloadTces" );
- *           // break from do loop if error occured
- *           break;
- *       }
- *
- *
- *       // Disable all TCEs
- *       l_err = TCE::utilDisableTces();
- *       if ( l_err )
- *       {
- *           TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
- *                      "Failed TCE::utilDisableTces" );
- *           // break from do loop if error occured
- *           break;
- *       }
- */
-
     } while(0);
 
     if( l_err )
diff --git a/src/usr/targeting/common/xmltohb/attribute_types.xml b/src/usr/targeting/common/xmltohb/attribute_types.xml
index fd2ff7946..96b02e053 100755
--- a/src/usr/targeting/common/xmltohb/attribute_types.xml
+++ b/src/usr/targeting/common/xmltohb/attribute_types.xml
@@ -7586,4 +7586,58 @@ Selects which voltage level to place the Core and ECO domain PFETs upon Winkle e
   </hwpfToHbAttrMap>
 </attribute>
 
+<attribute>
+  <id>USE_TCES_FOR_DMAS</id>
+  <description>
+<!-- @TODO RTC 168745 Remove This Attribute -->
+      Specifies whether or not the FSP is ready for Hostboot to enable TCEs
+      for DMAs for the given IPL.  This is a temporary attribute which will
+      be used to phase-in TCE support.
+      If 0, DO NOT use TCEs;
+      If 1, use TCEs
+  </description>
+  <simpleType>
+      <uint8_t>
+        <default>0</default>
+      </uint8_t>
+  </simpleType>
+  <persistency>non-volatile</persistency>
+  <readable/>
+  <writeable/>
+</attribute>
+
+<attribute>
+  <id>TCE_START_TOKEN_FOR_PAYLOAD</id>
+  <description>
+      Specifies which TCE Token the FSP should start with to transfer the
+      PAYLOAD into system memory.
+      If 0xFFFFFFFF, then invalid; otherwise, valid.
+  </description>
+  <simpleType>
+      <uint32_t>
+        <default>0xFFFFFFFF</default>
+      </uint32_t>
+  </simpleType>
+  <persistency>volatile</persistency>
+  <readable/>
+  <writeable/>
+</attribute>
+
+<attribute>
+  <id>TCE_START_TOKEN_FOR_HDAT</id>
+  <description>
+      Specifies which TCE Token the FSP should start with to transfer the
+      HDAT section into system memory.
+      If 0xFFFFFFFF, then invalid; otherwise, valid.
+  </description>
+  <simpleType>
+      <uint32_t>
+        <default>0xFFFFFFFF</default>
+      </uint32_t>
+  </simpleType>
+  <persistency>volatile</persistency>
+  <readable/>
+  <writeable/>
+</attribute>
+
 </attributes>
diff --git a/src/usr/targeting/common/xmltohb/target_types.xml b/src/usr/targeting/common/xmltohb/target_types.xml
index 978e79238..800751b69 100644
--- a/src/usr/targeting/common/xmltohb/target_types.xml
+++ b/src/usr/targeting/common/xmltohb/target_types.xml
@@ -907,6 +907,9 @@
   <attribute><id>MRW_MBA_CACHELINE_INTERLEAVE_MODE_CONTROL</id></attribute>
   <attribute><id>MAX_DMI_PER_PROC</id></attribute>
   <attribute><id>PIB_I2C_REFCLOCK</id></attribute>
+  <attribute><id>USE_TCES_FOR_DMAS</id></attribute>
+  <attribute><id>TCE_START_TOKEN_FOR_PAYLOAD</id></attribute>
+  <attribute><id>TCE_START_TOKEN_FOR_HDAT</id></attribute>
   <!--fsp requirement-->
   <attribute>
     <id>BOOT_FREQ_MHZ</id>
diff --git a/src/usr/util/utilmclmgr.C b/src/usr/util/utilmclmgr.C
index 79e527621..824df42b7 100644
--- a/src/usr/util/utilmclmgr.C
+++ b/src/usr/util/utilmclmgr.C
@@ -33,6 +33,7 @@
 #include <secureboot/trustedbootif.H>
 #include <runtime/preverifiedlidmgr.H>
 #include <limits.h>
+#include <util/utiltce.H>
 
 namespace MCL
 {
@@ -364,17 +365,27 @@ errlHndl_t MasterContainerLidMgr::processComponent(const ComponentID& i_compId,
 
     do {
 
-    // @TODO RTC:125304 Provide support for a preloaded PHYP
-    // Check if Component is POWERVM
+    // Check if Component is POWERVM (aka PHYP)
     bool isPhypComp = (i_compId == g_PowervmCompId) ? true : false;
-    // Still skip PHYP if (!isTCEmode)
-    if (isPhypComp)
+
+    // @TODO RTC:125304 Provide support for a preloaded PHYP
+    // Check if Component is POWERVM (PHYP) and still skip if (!isTCEmode)
+    if ( isPhypComp )
     {
-        // Skip Lid loading if PHYP component and in TCEmode
-        l_skipLoad = true;
-        // Only Skip PHYP if not in TCEmode
-        UTIL_FT("MasterContainerLidMgr::processComponent skipping POWERVM compoenent");
-        break;
+        if (TCE::utilUseTcesForDmas())
+        {
+            // Skip loading, but still process POWERVM (PHYP) component
+            l_skipLoad = true;
+            UTIL_FT("MasterContainerLidMgr::processComponent skip load but processing POWERVM component");
+        }
+        else
+        {
+            // Skip Lid loading and processing of POWERVM (PHYP) component
+            // if NOT in TCEmode
+            l_skipLoad = true;
+            UTIL_FT("MasterContainerLidMgr::processComponent skipping POWERVM component completely");
+            break;  // break from do-while to skip processing
+        }
     }
 
     // Only process compoenents if they are marked PRE_VERIFY
diff --git a/src/usr/util/utiltcemgr.C b/src/usr/util/utiltcemgr.C
index c1a030b65..54ce222f1 100644
--- a/src/usr/util/utiltcemgr.C
+++ b/src/usr/util/utiltcemgr.C
@@ -47,6 +47,7 @@
 #include <pnor/pnorif.H>
 #include <targeting/common/targetservice.H>
 #include <devicefw/userif.H>
+#include <initservice/initserviceif.H>
 
 trace_desc_t* g_trac_tce = nullptr;
 TRAC_INIT(&g_trac_tce, UTILTCE_TRACE_NAME, 4*KILOBYTE);
@@ -127,38 +128,12 @@ errlHndl_t getPayloadAddrAndSize(uint64_t& o_addr, size_t& o_size)
 {
     errlHndl_t errl = nullptr;
 
-    o_addr=0x0;
-    o_size=0x0;
-
-    do{
-
-    // Get Target Service and the system target to get ATTR_PAYLOAD_BASE
-    TARGETING::TargetService& tS = TARGETING::targetService();
-    TARGETING::Target* sys = nullptr;
-    (void) tS.getTopLevelTarget( sys );
-    assert(sys, "getPayloadAddrAndSize() system target is NULL");
-
-    o_addr = sys->getAttr<TARGETING::ATTR_PAYLOAD_BASE>()
-             * MEGABYTE; // Attribute value in MB
-
-    assert((o_addr % PAGESIZE) == 0, "getPayloadAddrAndSize(): o_addr=0x%.16llX not on page boundary", o_addr);
-
-    // Get PAYLOAD PNOR section info from PNOR RP
-    PNOR::SectionInfo_t payloadInfo;
-    errl = getSectionInfo( PNOR::PAYLOAD, //pnorSectionId,
-                           payloadInfo );
-
-    if(errl)
-    {
-        TRACFCOMP( g_trac_tce, ERR_MRK"getPayloadAddrAndSize() Error calling getSectionInfo() rc=0x%.4X", errl->reasonCode() );
-        break;
-    }
-
-    o_size = TCE_PAYLOAD_SIZE;
-
-    } while(0);
+    // Move PAYLOAD to Preverification Location
+    o_addr = MCL_TMP_ADDR;
+    o_size = MCL_TMP_SIZE;
 
     TRACFCOMP( g_trac_tce,EXIT_MRK"getPayloadAddrAndSize(): o_addr=0x%.16llX, o_size=0x%.16llX", o_addr, o_size);
+
     return errl;
 }
 
@@ -180,6 +155,8 @@ errlHndl_t utilSetupPayloadTces(void)
         break;
     }
 
+    // @TODO RTC 168745 - Update Interface to Return Table Position/aka Token
+    uint32_t token = 0;
     errl = utilAllocateTces(addr, size);
     if (errl)
     {
@@ -187,6 +164,15 @@ errlHndl_t utilSetupPayloadTces(void)
         break;
     }
 
+    // Set attribute to tell FSP that Payload has been setup at the start of the TCE Table
+    // Get Target Service and the system target to set TCE_START_TOKEN_FOR_PAYLOAD
+    TARGETING::TargetService& tS = TARGETING::targetService();
+    TARGETING::Target* sys = nullptr;
+    (void) tS.getTopLevelTarget( sys );
+    assert(sys, "utilSetupPayloadTces() system target is NULL");
+
+    sys->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_PAYLOAD>(token);
+
     } while(0);
 
     TRACFCOMP(g_trac_tce,EXIT_MRK"utilSetupPayloadTces(): Address=0x%.16llX, size=0x%X, errl_rc=0x%X", addr, size, ERRL_GETRC_SAFE(errl));
@@ -991,7 +977,7 @@ errlHndl_t UtilTceMgr::disableTces(void)
 
     // If the HW was initialized to use TCEs then disable those settings
     // it needs to be released here
-    if (iv_isTceHwInitDone)
+    if (iv_isTceHwInitDone==true)
     {
         // Loop through the processors and clear the TCE-related registers
         // in the PSI Host Bridge
@@ -1056,7 +1042,7 @@ errlHndl_t UtilTceMgr::disableTces(void)
     }
     else
     {
-        TRACUCOMP(g_trac_tce,"UtilTceMgr::disableTces: No Need To Uninitialize HW: iv_isTceHwInitDone=%d", iv_isTceHwInitDone);
+        TRACFCOMP(g_trac_tce,"UtilTceMgr::disableTces: No Need To Uninitialize HW: iv_isTceHwInitDone=%d", iv_isTceHwInitDone);
     }
 
     // Cleanup TCE Table In Memory
@@ -1213,6 +1199,7 @@ errlHndl_t UtilTceMgr::mapPsiHostBridge(const TARGETING::Target * i_tgt,
 
         errl->collectTrace(UTILTCE_TRACE_NAME,KILOBYTE);
     }
+
     o_psihb_ptr = l_ptr;
 
     TRACUCOMP(g_trac_tce,EXIT_MRK"UtilTceMgr::mapPsiHostBridge: o_psihb_ptr=0x%.16llX, Psi Bridge Addr = 0x%.16llX, huid = 0x%.8X", o_psihb_ptr, PsiBridgeAddr, TARGETING::get_huid(i_tgt));
@@ -1272,6 +1259,57 @@ errlHndl_t UtilTceMgr::unmapPsiHostBridge(void *& io_psihb_ptr) const
     return errl;
 }
 
+
+/******************************************************/
+/* Miscellaneous Functions                            */
+/******************************************************/
+bool utilUseTcesForDmas(void)
+{
+    bool retVal = false;
+
+    if (INITSERVICE::spBaseServicesEnabled())
+    {
+        // @TODO RTC 168745 - Eventually this will default to true in all cases
+        // where was have a FSP
+
+        // Get Target Service and the system target to get ATTR_USE_TCES_FOR_DMA
+        TARGETING::TargetService& tS = TARGETING::targetService();
+        TARGETING::Target* sys = nullptr;
+        (void) tS.getTopLevelTarget( sys );
+        assert(sys, "utilUseTcesForDmas() system target is NULL");
+
+        retVal = sys->getAttr<TARGETING::ATTR_USE_TCES_FOR_DMAS>();
+    }
+
+    TRACFCOMP(g_trac_tce,INFO_MRK"utilUseTcesForDmas: %s",
+              retVal ? "TRUE" : "FALSE");
+
+    return retVal;
+}
+
+errlHndl_t utilEnableTcesWithoutTceTable(void)
+{
+    errlHndl_t errl = nullptr;
+
+    // Create local UtilTceMgr with default TCE table address but with a size
+    // of zero so that all entries are invalid
+    // NOTE: memory at TCE Table Address is initialized to 0 as part of IPL and
+    //       all zero creates an invalid TCE entry
+    UtilTceMgr tceMgr(TCE_TABLE_ADDR, 0);
+
+    // Call initTceInHdw
+    errl = tceMgr.initTceInHdw();
+
+    if (errl)
+    {
+        TRACFCOMP(g_trac_tce,"utilEnableTcesWithoutTceTable(): initTceInHdw() "
+                  "failed with rc=0x%X", ERRL_GETRC_SAFE(errl));
+    }
+
+    return errl;
+
+}
+
 }; // namespace TCE
 
 #endif
diff --git a/src/usr/util/utiltcemgr.H b/src/usr/util/utiltcemgr.H
index 3f8d2b95c..b77f45023 100644
--- a/src/usr/util/utiltcemgr.H
+++ b/src/usr/util/utiltcemgr.H
@@ -148,15 +148,6 @@ class UtilTceMgr
 
 
     /**
-     * @brief Responsible for setting up the Processors to point to the TCE
-     *         Table
-     *
-     * @return errlHndl_t - Return error log if unsuccessful
-     *
-     */
-    errlHndl_t initTceInHdw();
-
-    /**
      * @brief Helper function to Memory Map PSI Host Bridge
      *
      * @param[in]  i_tgt       Pointer to Processor Target that is associated
@@ -275,6 +266,15 @@ class UtilTceMgr
      */
     errlHndl_t disableTces(void);
 
+    /**
+     * @brief Responsible for setting up the Processors to point to the TCE
+     *         Table
+     *
+     * @return errlHndl_t - Return error log if unsuccessful
+     *
+     */
+    errlHndl_t initTceInHdw();
+
 }; // class UtilTceMgr
 
     /**