Fix Initializing HW To Point At An Invalid TCE Table in Istep 10.6

This fixes a bug where the HW was not initialized properly to
point at an invalid TCE Table in istep 10.6.  This is required such
that the FSP cannot DMA into memory without Hostboot control.

Change-Id: I41ad102a4c05bfede5062f6082cbfa9041c0f1cf
RTC:168745
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/50652
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Stephen M. Cprek <smcprek@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

4 files changed, 35 insertions, 56 deletions

diff --git a/src/include/usr/util/utiltce.H b/src/include/usr/util/utiltce.H
index b0aaf48ae..f77d005f8 100644
--- a/src/include/usr/util/utiltce.H
+++ b/src/include/usr/util/utiltce.H
@@ -56,9 +56,6 @@ namespace TCE
  *                               FSP uses the token as a DMA Address.
  *                               Each TCE entry maps to a pagesize of memory.
  *
- * Note:  First time this is called will cause TCE Table to be created and
- *        the Processors to be setup to point at the TCE Table
- *
  * @return errlHndl_t - Return Error Handle if failed
  *
  */
@@ -130,6 +127,10 @@ bool utilUseTcesForDmas(void);
  *        This prevents the FSP from using unsecure DMAs during the IPL until
  *        Hostboot sets up the TCE Table with proper TCEs.
  *
+ * @note  This API only works once and only when called before any other TCE
+ *        Setup/Allocate functions as its logic only works the first time the
+ *        UtilTceMgr Singleton class is created.
+ *
  * @return errlHndl_t - Return Error Handle if failed
  *
  */
diff --git a/src/usr/isteps/istep21/call_host_runtime_setup.C b/src/usr/isteps/istep21/call_host_runtime_setup.C
index fbf00c458..bb6f4a23d 100644
--- a/src/usr/isteps/istep21/call_host_runtime_setup.C
+++ b/src/usr/isteps/istep21/call_host_runtime_setup.C
@@ -189,7 +189,7 @@ errlHndl_t verifyAndMovePayload(void)
 
     TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
                 "verifyAndMovePayload(): Copy PAYLOAD from 0x%.16llX (va="
-                "0x%llX) to PAYLOAD_BASE = 0x%.16llX (va=%llX), size=0x%llX",
+                "0x%llX) to PAYLOAD_BASE = 0x%.16llX (va=0x%llX), size=0x%llX",
                 payload_tmp_phys_addr, payload_tmp_virt_addr, payloadBase,
                 payloadBase_virt_addr, payload_size);
 
diff --git a/src/usr/util/utiltcemgr.C b/src/usr/util/utiltcemgr.C
index 2424ee686..1f67606a9 100644
--- a/src/usr/util/utiltcemgr.C
+++ b/src/usr/util/utiltcemgr.C
@@ -258,6 +258,19 @@ UtilTceMgr::UtilTceMgr(const uint64_t i_tableAddr, const size_t i_tableSize)
 
     TRACUCOMP(g_trac_tce,"UtilTceMgr::UtilTceMgr: iv_tceTableVaAddr=0x%.16llX, iv_tceTablePhysAddr=0x%.16llX, iv_tceTableSize=0x%llX, iv_tceEntryCount=0x%X, iv_allocatedAddrs,size=%d", iv_tceTableVaAddr, iv_tceTablePhysAddr, iv_tceTableSize, iv_tceEntryCount, iv_allocatedAddrs.size());
 
+    // Initialize HW without Initializing Table so that FSP cannot DMA
+    // to memory without Hostboot control
+    auto errl = UtilTceMgr::initTceInHdw();
+    if (errl)
+    {
+        uint32_t errl_plid = errl->plid();
+        TRACFCOMP(g_trac_tce,"UtilTceMgr::UtilTceMgr initTceInHdw() failed with rc=0x%X, plid=0x%X. Shutting down",ERRL_GETRC_SAFE(errl), errl_plid);
+        errl->setSev(ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM);
+        errl->collectTrace(UTILTCE_TRACE_NAME,KILOBYTE);
+        errlCommit( errl, UTIL_COMP_ID );
+        INITSERVICE::doShutdown(errl_plid, true);
+    }
+
 };
 
 /**************************************************************************/
@@ -769,7 +782,7 @@ errlHndl_t UtilTceMgr::allocateTces(const uint64_t i_startingAddress,
                 tablePtr[index].writeAccess = 1;
                 tablePtr[index].readAccess = 1;
 
-                TRACDCOMP(g_trac_tce,INFO_MRK"UtilTceMgr::allocateTces: TCE Entry/Token[%d] (hex) = %llX", index, tablePtr[index]);
+                TRACDCOMP(g_trac_tce,INFO_MRK"UtilTceMgr::allocateTces: TCE Entry/Token[%d] (hex) = 0x%llX", index, tablePtr[index]);
             }
 
             // Save And Return Information about Allocated TCEs
@@ -929,7 +942,7 @@ errlHndl_t UtilTceMgr::deallocateTces(const uint32_t i_startingToken,
         }
 
         // Remove the entry from iv_allocatedAddrs even if 'isContiguous' issue
-        iv_allocatedAddrs.erase(startingIndex);
+        iv_allocatedAddrs.erase(i_startingToken);
 
         if (!isContiguous)
         {
@@ -1128,6 +1141,7 @@ UtilTceMgr::~UtilTceMgr()
     if (errl)
     {
         TRACFCOMP(g_trac_tce,"UtilTceMgr::~UtilTceMgr: disableTces Failed rc=0x%X. Committing plid=0x%X", ERRL_GETRC_SAFE(errl), ERRL_GETPLID_SAFE(errl));
+        errl->collectTrace(UTILTCE_TRACE_NAME,KILOBYTE);
         errlCommit( errl, UTIL_COMP_ID );
     }
 
@@ -1340,21 +1354,9 @@ errlHndl_t utilEnableTcesWithoutTceTable(void)
 {
     errlHndl_t errl = nullptr;
 
-    // @TODO RTC 168745 - Update to use Singleton UtilTceMgr
-    // Create local UtilTceMgr with default TCE table address but with a size
-    // of zero so that all entries are invalid
-    // NOTE: memory at TCE Table Address is initialized to 0 as part of IPL and
-    //       all zero creates an invalid TCE entry
-    UtilTceMgr tceMgr(TCE_TABLE_ADDR, 0);
-
-    // Call initTceInHdw
-    errl = tceMgr.initTceInHdw();
-
-    if (errl)
-    {
-        TRACFCOMP(g_trac_tce,"utilEnableTcesWithoutTceTable(): initTceInHdw() "
-                  "failed with rc=0x%X", ERRL_GETRC_SAFE(errl));
-    }
+    // This will call the constructor, which in turn will initialize the
+    // HW to point at a TCE Table with invalid entries
+    Singleton<UtilTceMgr>::instance();
 
     return errl;
 
diff --git a/src/usr/util/utiltcemgr.H b/src/usr/util/utiltcemgr.H
index aabf99d5a..9bb643dbf 100644
--- a/src/usr/util/utiltcemgr.H
+++ b/src/usr/util/utiltcemgr.H
@@ -165,6 +165,14 @@ class UtilTceMgr
      */
     errlHndl_t createTceTable();
 
+    /**
+     * @brief Responsible for setting up the Processors to point to the TCE
+     *         Table
+     *
+     * @return errlHndl_t - Return error log if unsuccessful
+     *
+     */
+    errlHndl_t initTceInHdw();
 
     /**
      * @brief Helper function to Memory Map PSI Host Bridge
@@ -243,23 +251,7 @@ class UtilTceMgr
     /**
      * @brief Responsible for allocating TCEs
      *
-     * @param[in] i_startingAddress - Starting address that the allocated TCEs
-     *                                will map to
-     *                                Needs to be page-aligned or will fail
-     * @param[in] i_size - Size of the address space that TCEs map to
-     *                     Size must be less than (512K * PAGESIZE) or will fail
-     *                     Assert if not greater than zero
-     * @param[out] o_startingToken - Starting Entry into the table.
-     * @param[out] o_startingToken - Key to the Starting Entry into the table.
-     *                               This is an offset into the array based on the
-     *                                 TCE Index * PAGESIZE.
-     *                               FSP uses the token as a DMA Address.
-     *                               Each TCE entry maps to a pagesize of memory.
-     *
-     * Note:  First time this is called will cause TCE table to be created and
-     *        the Processors to be setup to point at the TCE table
-     *
-     * @return errl - Return Error Handle if failed.
+     * - see utiltce.H for details
      *
      */
     errlHndl_t allocateTces(uint64_t  i_startingAddress,
@@ -269,13 +261,7 @@ class UtilTceMgr
     /**
      * @brief Responsible for deallocating TCEs
      *
-     * @param[in] i_startingToken  Token indicating the starting entry of the
-     *                             TCE Table to remove
-     * @param[in] i_size - Size of memory space to remove associated TCE entries
-     *                     Size must be less than (512K * PAGESIZE) or will fail
-     *                     Assert if not greater than zero
-     *
-     * @return errl - Return Error Handle if fatal failure occurred.
+     * - see utiltce.H for details
      *
      */
     errlHndl_t deallocateTces(uint32_t i_startingToken,
@@ -285,21 +271,11 @@ class UtilTceMgr
      * @brief Responsible for disabling TCEs on the system, including
      *        clearing the TCE table and disabling Processor settings
      *
-     * @return errl - Return Error Handle if fatal failure occurred.
+     * - see utiltce.H for details
      *
      */
     errlHndl_t disableTces(void);
 
-    /**
-     * @brief Responsible for setting up the Processors to point to the TCE
-     *         Table
-     *
-     * @return errlHndl_t - Return error log if unsuccessful
-     *
-     */
-    errlHndl_t initTceInHdw();
-
-
     /** Values related to tokens */
     enum tokenLabels
     {