Relocate bl to hb preserved data and page table in VMM

Relocate Page Manager Page Table to 256K alignment after preserved area
Simplify page manager initialize

Change-Id: Ic90584437fa68843a7ebe3818d48c3fe4f5157d8
RTC: 175114
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/42154
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

10 files changed, 179 insertions, 85 deletions

diff --git a/src/bootloader/bootloader.C b/src/bootloader/bootloader.C
index 5d5fe2681..d9b9b5eb7 100644
--- a/src/bootloader/bootloader.C
+++ b/src/bootloader/bootloader.C
@@ -127,7 +127,7 @@ namespace Bootloader{
         {
             // Add cases as additional versions are created
             default:
-                g_blData->blToHbData.version = BLTOHB_SECURE_OVERRIDES;
+                g_blData->blToHbData.version = BLTOHB_SIZE;
                 break;
         }
 
@@ -159,6 +159,9 @@ namespace Bootloader{
             // Set HBB header and size
             g_blData->blToHbData.hbbHeader = i_pHbbSrc;
             g_blData->blToHbData.hbbHeaderSize = PAGE_SIZE;
+
+            // Set Bootloader preceived size of structure
+            g_blData->blToHbData.sizeOfStructure = sizeof(BlToHbData);
         }
 
         // Place structure into proper location for HB to find
diff --git a/src/build/debug/Hostboot/BlData.pm b/src/build/debug/Hostboot/BlData.pm
index ddbbb7304..b269305d0 100644
--- a/src/build/debug/Hostboot/BlData.pm
+++ b/src/build/debug/Hostboot/BlData.pm
@@ -208,7 +208,7 @@ sub main
 
     my $blToHbAddr = $dataAddr + $dataOffset;
     my $blToHbAddrStr = sprintf("0x%08X", $blToHbAddr);
-    my $blToHbSize = 91;
+    my $blToHbSize = 99;
     my $blToHb = ::readData($blToHbAddr,$blToHbSize);
     my $blToHbData = formatData($blToHb);
     $dataOffset += ::alignUp($blToHbSize, 16);
diff --git a/src/include/bootloader/bootloader_data.H b/src/include/bootloader/bootloader_data.H
index a6f4654e1..289e1c20d 100644
--- a/src/include/bootloader/bootloader_data.H
+++ b/src/include/bootloader/bootloader_data.H
@@ -91,7 +91,7 @@ namespace Bootloader{
         // Object that will be stored where the SBE HB structure indicates
         BlToHbData blToHbData;
         uint8_t bl_reserved5[(512 - sizeof(BlToHbData)) % 16];
-        static_assert( sizeof(BlToHbData) == 91, "BlToHbData "
+        static_assert( sizeof(BlToHbData) == 99, "BlToHbData "
                        "size changed.  Check bootloader_data.H alignment.  "
                        "Fix BlData.pm processing.");
     } blData_t;
diff --git a/src/include/bootloader/bootloaderif.H b/src/include/bootloader/bootloaderif.H
index 25b3520e8..fc810dc4b 100644
--- a/src/include/bootloader/bootloaderif.H
+++ b/src/include/bootloader/bootloaderif.H
@@ -60,8 +60,15 @@ enum BlToHbDataVersion
     BLTOHB_SAB              = 0x0000000900000002,
     BLTOHB_MMIOBARS         = 0x0000000900000003,
     BLTOHB_SECURE_OVERRIDES = 0x0000000900000004,
+    BLTOHB_SIZE             = 0x0000000900000005,
 };
 
+enum
+{
+    // Arbitrary size larger than the current structure size 8 byte aligned.
+    // This will only be used when there is a BL/HB mismatch
+    INITIAL_BLTOHB_PADDED_SIZE = 256
+};
 
 /** @struct BlToHbData
  *  @brief  Shared data between bootloader and Hostboot.
@@ -81,7 +88,8 @@ struct BlToHbData
                    hbbHeaderSize(0), secureAccessBit(0),
                    xscomBAR(MMIO_GROUP0_CHIP0_XSCOM_BASE_ADDR),
                    lpcBAR(MMIO_GROUP0_CHIP0_LPC_BASE_ADDR),
-                   securityOverride(0), allowAttrOverrides(0) {}
+                   securityOverride(0), allowAttrOverrides(0),
+                   sizeOfStructure(0) {}
 
     // Simple way to tell if data is valid
     uint64_t eyeCatch;
@@ -111,6 +119,8 @@ struct BlToHbData
     bool securityOverride;
     // Secure Setting - Allow Attribute Overrides in Securemode
     bool allowAttrOverrides;
+    // Size of this structure (Use for backwards compatibility)
+    size_t sizeOfStructure;
 } __attribute__((packed));
 
 /**
diff --git a/src/include/kernel/bltohbdatamgr.H b/src/include/kernel/bltohbdatamgr.H
index 3e701b01b..6b6f8b831 100644
--- a/src/include/kernel/bltohbdatamgr.H
+++ b/src/include/kernel/bltohbdatamgr.H
@@ -93,6 +93,15 @@ class BlToHbDataManager
          */
         void initInvalid();
 
+        /**
+         * @brief Relocates preserved data to a location in cache that will not
+         *        be flushed
+         *        NOTE: Asserts that it's only called during initialization
+         *
+         * @return N/A
+         */
+        void relocatePreservedArea();
+
         /*
          * @brief Returns internal branchtable offset
          *
@@ -191,6 +200,13 @@ class BlToHbDataManager
          * @return size_t     XSCP< BAR
          */
         const uint64_t getXscomBAR() const;
+
+        /*
+         * @brief Returns internal BlToHbData size
+         *
+         * @return size_t     BlToHbData size
+         */
+        const size_t getBlToHbDataSize() const;
 };
 
 // Extern global instance of the class
diff --git a/src/include/kernel/vmmmgr.H b/src/include/kernel/vmmmgr.H
index 9c437fe23..5cc370e69 100644
--- a/src/include/kernel/vmmmgr.H
+++ b/src/include/kernel/vmmmgr.H
@@ -45,21 +45,14 @@ class VmmManager
             INITIAL_MEM_SIZE = 4*MEGABYTE,
 
             // Place the page table at the top side of the cache, 256k in size.
-            INITIAL_PT_OFFSET = INITIAL_MEM_SIZE - 1*MEGABYTE,
             PTSIZE = 256*KILOBYTE,
+            PT_ALIGNMENT = 256*KILOBYTE,
 
             // Put the DMA Pages just after the Page Table
             MBOX_DMA_PAGES = 64,    // must be <= 64
             MBOX_DMA_PAGESIZE = (1 * KILOBYTE),
             MBOX_DMA_SIZE = MBOX_DMA_PAGES * MBOX_DMA_PAGESIZE,
 
-            /** We need to reserve a hole in heap memory for the page table,
-             *  etc.  Use these constants to define the hole. */
-            FIRST_RESERVED_PAGE = INITIAL_PT_OFFSET,
-            END_RESERVED_PAGE = INITIAL_PT_OFFSET + PTSIZE,
-
-            BLTOHB_DATA_START = END_RESERVED_PAGE,
-
             // Tells processor to ignore HRMOR
             FORCE_PHYS_ADDR = 0x8000000000000000,
         };
@@ -220,6 +213,20 @@ class VmmManager
          */
         static uint64_t pageTableOffset();
 
+        /** @fn BlToHbPreserveDataOffset()
+         *  @brief   Gets starting address of BltoHB preserved data area
+         *
+         *  @return uint64_t - starting address of VMM area
+         */
+        static uint64_t BlToHbPreserveDataOffset();
+
+        /** @fn endPreservedOffset()
+         *  @brief   Gets ending address of preserved data area
+         *
+         *  @return uint64_t - ending address of VMM preserved area
+         */
+        static uint64_t endPreservedOffset();
+
     protected:
         VmmManager();
         ~VmmManager() {};
@@ -282,6 +289,12 @@ class VmmManager
         /** See pageTableOffset */
         uint64_t _pageTableOffset() const;
 
+        /** See BlToHbPreserveDataOffset */
+        uint64_t _BlToHbPreserveDataOffset() const;
+
+        /** See endPreservedOffset */
+        uint64_t _endPreservedOffset() const;
+
     public:
         friend class Block;
         friend class StackSegment;
diff --git a/src/kernel/bltohbdatamgr.C b/src/kernel/bltohbdatamgr.C
index 1fa211381..537958214 100644
--- a/src/kernel/bltohbdatamgr.C
+++ b/src/kernel/bltohbdatamgr.C
@@ -28,6 +28,7 @@
 #include <assert.h>
 #include <arch/memorymap.H>
 #include <bootloader/bootloaderif.H>
+#include <kernel/vmmmgr.H>
 
 // Global and only BlToHbDataManager instance
 BlToHbDataManager g_BlToHbDataManager;
@@ -76,6 +77,10 @@ void BlToHbDataManager::print() const
         printkd("-- HBB header Addr = 0x%lX Size = 0x%lX\n", getHbbHeaderAddr(),
                iv_data.hbbHeaderSize);
         printkd("-- Reserved Size = 0x%lX\n", iv_preservedSize);
+        if(iv_data.version >= Bootloader::BLTOHB_SIZE)
+        {
+            printkd("-- Size of structure = 0x%lX\n", iv_data.sizeOfStructure);
+        }
         printkd("\n");
     }
 }
@@ -166,13 +171,28 @@ printk("lpc=%lX, xscom=%lX\n", i_data.lpcBAR, i_data.xscomBAR );
 printk("lpc=%lX, xscom=%lX, iv_data=%p\n", iv_data.lpcBAR, iv_data.xscomBAR,
        static_cast<void *>(&iv_data) );
 
+    // Check if bootloader advertised the size of the structure it saw;
+    // otherwise use the default padded size
+    if(iv_data.version >= Bootloader::BLTOHB_SIZE)
+    {
+        iv_data.sizeOfStructure = i_data.sizeOfStructure;
+    }
+    else
+    {
+        iv_data.sizeOfStructure = Bootloader::INITIAL_BLTOHB_PADDED_SIZE;
+    }
 
     // Size of data that needs to be preserved and pinned.
     iv_preservedSize = ALIGN_PAGE(iv_data.secureRomSize +
                                  iv_data.hwKeysHashSize +
                                  iv_data.hbbHeaderSize );
+
+    // Move preserved content to a location free from cache clearing
+    relocatePreservedArea();
+
     iv_initialized = true;
     iv_dataValid = true;
+
     print();
 }
 
@@ -195,6 +215,46 @@ void BlToHbDataManager::initInvalid ()
     print();
 }
 
+void BlToHbDataManager::relocatePreservedArea()
+{
+    // Allow call this within the initializer
+    if (iv_initialized)
+    {
+        printk("E> BlToHbDataManager relocatePreservedArea called outside initializer\n");
+        kassert(!iv_initialized);
+    }
+    // Ensure the pointers were initialized
+    kassert(iv_data.secureRom!=nullptr);
+    kassert(iv_data.hwKeysHash!=nullptr);
+    kassert(iv_data.hbbHeader!=nullptr);
+
+    // Get destination location that will be preserved by the pagemgr
+    auto l_pBltoHbDataStart = reinterpret_cast<uint8_t *>(
+                                    VmmManager::BlToHbPreserveDataOffset());
+    // Copy in SecureRom
+    memcpy(l_pBltoHbDataStart,
+           iv_data.secureRom,
+           iv_data.secureRomSize);
+    // Change pointer to new location and increment
+    iv_data.secureRom = l_pBltoHbDataStart;
+    l_pBltoHbDataStart += iv_data.secureRomSize;
+
+    // Copy in HW keys' Hash
+    memcpy(l_pBltoHbDataStart,
+           iv_data.hwKeysHash,
+           iv_data.hwKeysHashSize);
+    // Change pointer to new location and increment
+    iv_data.hwKeysHash = l_pBltoHbDataStart;
+    l_pBltoHbDataStart += iv_data.hwKeysHashSize;
+
+    // Copy in HBB header
+    memcpy(l_pBltoHbDataStart,
+           iv_data.hbbHeader,
+           iv_data.hbbHeaderSize);
+    // Change pointer to new location
+    iv_data.hbbHeader = l_pBltoHbDataStart;
+}
+
 const uint64_t BlToHbDataManager::getBranchtableOffset() const
 {
     return iv_data.branchtableOffset;
@@ -286,3 +346,8 @@ const uint64_t BlToHbDataManager::getXscomBAR() const
     return reinterpret_cast<uint64_t>(iv_data.xscomBAR);
 }
 
+const size_t BlToHbDataManager::getBlToHbDataSize() const
+{
+    return iv_data.sizeOfStructure;
+}
+
diff --git a/src/kernel/kernel.C b/src/kernel/kernel.C
index 5e97ebb4c..d5952374a 100644
--- a/src/kernel/kernel.C
+++ b/src/kernel/kernel.C
@@ -82,38 +82,8 @@ int main()
     if ( Bootloader::BlToHbDataValid(l_pBltoHbData) )
     {
         printk("BL to HB comm valid\n");
-
-        // Make copy of structure so to not modify original pointers
-        auto l_blToHbDataCopy = *l_pBltoHbData;
-
-        // Get destination location that will be preserved by the pagemgr
-        auto l_pBltoHbDataStart = reinterpret_cast<uint8_t *>(
-                                                 VmmManager::BLTOHB_DATA_START);
-        // Copy in SecureRom
-        memcpy(l_pBltoHbDataStart,
-               l_blToHbDataCopy.secureRom,
-               l_blToHbDataCopy.secureRomSize);
-        // Change pointer to new location and increment
-        l_blToHbDataCopy.secureRom = l_pBltoHbDataStart;
-        l_pBltoHbDataStart += l_blToHbDataCopy.secureRomSize;
-
-        // Copy in HW keys' Hash
-        memcpy(l_pBltoHbDataStart,
-               l_blToHbDataCopy.hwKeysHash,
-               l_blToHbDataCopy.hwKeysHashSize);
-        // Change pointer to new location and increment
-        l_blToHbDataCopy.hwKeysHash = l_pBltoHbDataStart;
-        l_pBltoHbDataStart += l_blToHbDataCopy.hwKeysHashSize;
-
-        // Copy in HBB header
-        memcpy(l_pBltoHbDataStart,
-               l_blToHbDataCopy.hbbHeader,
-               l_blToHbDataCopy.hbbHeaderSize);
-        // Change pointer to new location
-        l_blToHbDataCopy.hbbHeader = l_pBltoHbDataStart;
-
         // Initialize Secureboot Data class
-        g_BlToHbDataManager.initValid(l_blToHbDataCopy);
+        g_BlToHbDataManager.initValid(*l_pBltoHbData);
     }
     else
     {
diff --git a/src/kernel/pagemgr.C b/src/kernel/pagemgr.C
index 02e8ce710..bdc8244f2 100644
--- a/src/kernel/pagemgr.C
+++ b/src/kernel/pagemgr.C
@@ -202,61 +202,55 @@ PageManager::PageManager()
 
 void PageManager::_initialize()
 {
-    typedef PageManagerCore::page_t page_t;
-    uint64_t totalPages = 0;
-
-    page_t* startAddr = reinterpret_cast<page_t*>(firstPageAddr());
-    printk("PageManager starts at %p\n", startAddr);
+    printk("Hostboot base image ends at 0x%lX...\n", firstPageAddr());
 
-    // Populate cache lines from end of HBB to PT offset and add to heap
-    uint64_t startBlock = reinterpret_cast<uint64_t>(startAddr);
-    uint64_t endBlock = VmmManager::INITIAL_PT_OFFSET;
+    uint64_t totalPages = 0;
+    // Extend memory footprint to half the cache
+    // There is a preserved area after the base image and boot loader to HB
+    // communication area. The page table must be 256KB aligned, so it is
+    // likely to not be flush against the preserved area end.
+    // Example:
+    //      [HBB max size][BlToHBData][8 byte aligned]
+    //      [128 byte aligned Preserved-area][256K aligned Page Table]
+    uint64_t l_endPreservedArea = VmmManager::endPreservedOffset();
+    uint64_t l_endInitCache = VmmManager::INITIAL_MEM_SIZE;
+    uint64_t l_pageTableOffset = VmmManager::pageTableOffset();
+    uint64_t l_endPageTable = l_pageTableOffset + VmmManager::PTSIZE;
+
+    printk("PageManager end of preserved area at 0X%lX\n", l_endPreservedArea);
+    printk("PageManager page table offset at 0X%lX\n", l_pageTableOffset);
+
+    // Populate half the cache after the preserved area
     KernelMisc::populate_cache_lines(
-                reinterpret_cast<uint64_t*>(startBlock),
-                reinterpret_cast<uint64_t*>(endBlock));
-
-    uint64_t pages = (endBlock - startBlock) / PAGESIZE;
-    iv_heap.addMemory(startBlock, pages);
-    totalPages += pages;
+                                reinterpret_cast<uint64_t*>(l_endPreservedArea),
+                                reinterpret_cast<uint64_t*>(l_endInitCache));
 
-    // Populate cache lines of PT
-    startBlock = VmmManager::INITIAL_PT_OFFSET;
-    endBlock = VmmManager::INITIAL_PT_OFFSET + VmmManager::PTSIZE;
-    KernelMisc::populate_cache_lines(reinterpret_cast<uint64_t*>(startBlock),
-                                     reinterpret_cast<uint64_t*>(endBlock));
-
-    // Populate cachelines from end of Preserved read (PT + securebood data) to
-    // 4MB and add to heap
-    // Add on secureboot data size to end of reserved space
-    size_t securebootDataSize = 0;
-    if (g_BlToHbDataManager.isValid())
+    // Allocate heap memory between end of preserved area and start of page
+    // table, if necessary
+    uint64_t pages = 0;
+    if ( (l_pageTableOffset - l_endPreservedArea) > 0 )
     {
-        securebootDataSize = g_BlToHbDataManager.getPreservedSize();
+        pages = (l_pageTableOffset - l_endPreservedArea) / PAGESIZE;
+        iv_heap.addMemory(l_endPreservedArea, pages);
+        totalPages += pages;
     }
-    size_t l_endReservedPage = VmmManager::BLTOHB_DATA_START
-                               + securebootDataSize;
-    startBlock = l_endReservedPage;
-    endBlock = VmmManager::INITIAL_MEM_SIZE;
-    KernelMisc::populate_cache_lines(
-        reinterpret_cast<uint64_t*>(startBlock),
-        reinterpret_cast<uint64_t*>(endBlock));
-
-    pages = (endBlock - startBlock) / PAGESIZE;
-    iv_heap.addMemory(startBlock, pages);
+    // After the Page table
+    pages = (l_endInitCache - l_endPageTable) / PAGESIZE;
+    iv_heap.addMemory(l_endPageTable, pages);
     totalPages += pages;
 
     printk("%ld pages.\n", totalPages);
 
-    // Reserve pages for the kernel.
-    iv_heapKernel.addMemory(reinterpret_cast<uint64_t>(
-                              iv_heap.allocatePage(KERNEL_HEAP_RESERVED_PAGES)),
-                            KERNEL_HEAP_RESERVED_PAGES);
-
     // Statistics
     iv_pagesTotal = totalPages;
     iv_pagesAvail = totalPages;
     cv_low_page_count = totalPages;
 
+    // Reserve pages for the kernel.
+    iv_heapKernel.addMemory(reinterpret_cast<uint64_t>(
+                              iv_heap.allocatePage(KERNEL_HEAP_RESERVED_PAGES)),
+                            KERNEL_HEAP_RESERVED_PAGES);
+
     KernelMemState::setMemScratchReg(KernelMemState::MEM_CONTAINED_L3,
                                      KernelMemState::HALF_CACHE);
 }
diff --git a/src/kernel/vmmmgr.C b/src/kernel/vmmmgr.C
index 9902a0527..553fa3d34 100644
--- a/src/kernel/vmmmgr.C
+++ b/src/kernel/vmmmgr.C
@@ -33,6 +33,8 @@
 #include <kernel/stacksegment.H>
 #include <kernel/devicesegment.H>
 #include <config.h>
+#include <kernel/bltohbdatamgr.H>
+#include <util/align.H>
 
 
 extern void* data_load_address;
@@ -312,5 +314,26 @@ uint64_t VmmManager::pageTableOffset()
 
 uint64_t VmmManager::_pageTableOffset() const
 {
-    return INITIAL_PT_OFFSET;
+    return ALIGN_X(_endPreservedOffset(), PT_ALIGNMENT);
+}
+
+uint64_t VmmManager::BlToHbPreserveDataOffset()
+{
+    return Singleton<VmmManager>::instance()._BlToHbPreserveDataOffset();
+}
+
+uint64_t VmmManager::_BlToHbPreserveDataOffset() const
+{
+    return  ALIGN_8(MAX_HBB_SIZE + g_BlToHbDataManager.getBlToHbDataSize());
+}
+
+uint64_t VmmManager::endPreservedOffset()
+{
+    return Singleton<VmmManager>::instance()._endPreservedOffset();
+}
+
+uint64_t VmmManager::_endPreservedOffset() const
+{
+    return ALIGN_PAGE(_BlToHbPreserveDataOffset() +
+                      g_BlToHbDataManager.getPreservedSize());
 }