Move mbox DMA buffer to unsecure memory.

RTC: 64763
Change-Id: Ie910a57ca96bc6fc673097dbaf72e7df469b2017
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/4803
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

7 files changed, 95 insertions, 57 deletions

diff --git a/src/kernel/pagemgr.C b/src/kernel/pagemgr.C
index c402c51bb..3c05fc6ab 100644
--- a/src/kernel/pagemgr.C
+++ b/src/kernel/pagemgr.C
@@ -211,11 +211,6 @@ void PageManager::_initialize()
 
         uint64_t endBlock = ALIGN_MEGABYTE_DOWN(currentBlock) + 512*KILOBYTE;
 
-        // Populate L3 cache lines for this chunk.
-        KernelMisc::populate_cache_lines(
-            reinterpret_cast<uint64_t*>(currentBlock),
-            reinterpret_cast<uint64_t*>(endBlock));
-
         // Adjust address to compensate for reserved hole and add to
         // heap...
 
@@ -246,16 +241,27 @@ void PageManager::_initialize()
             // Hole is in the middle... yuck.
             else
             {
-                uint64_t pages =
-                    (VmmManager::FIRST_RESERVED_PAGE - currentBlock) / PAGESIZE;
+                uint64_t hole_end =
+                    (VmmManager::FIRST_RESERVED_PAGE - currentBlock);
+
+                // Populate L3 for the first part of the chunk.
+                KernelMisc::populate_cache_lines(
+                    reinterpret_cast<uint64_t*>(currentBlock),
+                    reinterpret_cast<uint64_t*>(hole_end));
 
-                iv_heap.addMemory(currentBlock, pages);
-                totalPages += pages;
+                // Add it to the heap.
+                iv_heap.addMemory(currentBlock, hole_end / PAGESIZE);
+                totalPages += (hole_end / PAGESIZE);
 
                 currentBlock = VmmManager::END_RESERVED_PAGE;
             }
         }
 
+        // Populate L3 cache lines for this chunk.
+        KernelMisc::populate_cache_lines(
+            reinterpret_cast<uint64_t*>(currentBlock),
+            reinterpret_cast<uint64_t*>(endBlock));
+
         uint64_t pages = (endBlock - currentBlock) / PAGESIZE;
 
         iv_heap.addMemory(currentBlock, pages);
@@ -265,6 +271,12 @@ void PageManager::_initialize()
 
     } while (reinterpret_cast<page_t*>(currentBlock) != endAddr);
 
+    // Ensure HW page table area is erased / populated.
+    KernelMisc::populate_cache_lines(
+        reinterpret_cast<uint64_t*>(VmmManager::INITIAL_PT_OFFSET),
+        reinterpret_cast<uint64_t*>(VmmManager::INITIAL_PT_OFFSET +
+                                    VmmManager::PTSIZE));
+
     printk("%ld pages.\n", totalPages);
 
     // Reserve pages for the kernel.
diff --git a/src/usr/mbox/mbox_dma_buffer.C b/src/usr/mbox/mbox_dma_buffer.C
index 2580aaab3..667c922de 100644
--- a/src/usr/mbox/mbox_dma_buffer.C
+++ b/src/usr/mbox/mbox_dma_buffer.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012                   */
+/* COPYRIGHT International Business Machines Corp. 2012,2013              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -26,7 +26,8 @@
 #include <util/align.H>
 #include <trace/interface.H>
 #include <kernel/pagemgr.H>
-
+#include <kernel/misc.H>
+#include <targeting/common/targetservice.H>
 
 #define ALIGN_DMAPAGE(u) (((u) + (VmmManager::MBOX_DMA_PAGESIZE-1)) & \
                           ~(VmmManager::MBOX_DMA_PAGESIZE-1))
@@ -41,7 +42,7 @@ DmaBuffer::DmaBuffer() :
     iv_dir(makeMask(VmmManager::MBOX_DMA_PAGES))
 {
     iv_head = reinterpret_cast<void*>(VmmManager::MBOX_DMA_ADDR);
-    iv_phys_head = mm_virt_to_phys(iv_head);
+    initPhysicalArea(iv_head, iv_phys_head);
 }
 
 
@@ -51,16 +52,16 @@ DmaBuffer::~DmaBuffer()
 
 void DmaBuffer::release(void * i_buffer, size_t i_size)
 {
-    if(!i_buffer) 
+    if(!i_buffer)
     {
         return;
     }
 
     // Make sure this buffer falls inside the DMA space
     // If not then it's not a DMA buffer - exit.
-    if(i_buffer < iv_head || 
-       i_buffer >= (static_cast<uint8_t*>(iv_head) + 
-                    (VmmManager::MBOX_DMA_PAGES * 
+    if(i_buffer < iv_head ||
+       i_buffer >= (static_cast<uint8_t*>(iv_head) +
+                    (VmmManager::MBOX_DMA_PAGES *
                      VmmManager::MBOX_DMA_PAGESIZE)))
     {
         TRACDCOMP(g_trac_mbox,
@@ -100,7 +101,7 @@ void * DmaBuffer::getBuffer(uint64_t & io_size)
     // make a mask for the requested size. Instead, getBuffer will just return
     // NULL if the requested size is larger than the total possible DMA buffer
     // space.
-    if(io_size == 0 || 
+    if(io_size == 0 ||
        io_size > (VmmManager::MBOX_DMA_PAGES * VmmManager::MBOX_DMA_PAGESIZE))
     {
         io_size = 0;
@@ -146,7 +147,7 @@ uint64_t DmaBuffer::makeMask(uint64_t i_size)
     assert(i_size <= MAX_MASK_SIZE);
     uint64_t mask = 0;
 
-    // For some reason (1ul << 64) returns 1, not zero 
+    // For some reason (1ul << 64) returns 1, not zero
     // The math function pow() converts things to float then back again - bad
     if(i_size < MAX_MASK_SIZE)
     {
@@ -159,3 +160,39 @@ uint64_t DmaBuffer::makeMask(uint64_t i_size)
     return mask;
 }
 
+void DmaBuffer::initPhysicalArea(void*& io_addr, uint64_t& o_phys)
+{
+    // Get the original physical address (includes HRMOR).
+    o_phys = mm_virt_to_phys(io_addr);
+
+    // Remove original address from VMM maps.
+    mm_set_permission(io_addr, VmmManager::MBOX_DMA_SIZE, NO_ACCESS);
+
+    // Find the amount of the node offset.
+    TARGETING::Target * sys = NULL;
+    TARGETING::targetService().getTopLevelTarget( sys );
+    assert(sys != NULL);
+    uint64_t hrmor_base =
+        sys->getAttr<TARGETING::ATTR_HB_HRMOR_NODAL_BASE>();
+
+    // Move the physical address to the start of the node (unsecure) and
+    // add on the DMA buffer offset inside the node.
+    o_phys &= ~(hrmor_base-1);
+    o_phys += VmmManager::MBOX_DMA_ADDR;
+
+    // Allocate a new VMM block for the buffer.
+    io_addr = mm_block_map(reinterpret_cast<void*>(o_phys),
+                           VmmManager::MBOX_DMA_SIZE);
+        // Note: We do not plan on deleting this block, even when the buffer
+        //       is destructed, because we have fundamentally changed the
+        //       underlying memory by populating the addresses in a different
+        //       physical location (see populate_cache_lines call below).
+
+    // Populate cache lines in unsecure memory.
+    KernelMisc::populate_cache_lines(
+        reinterpret_cast<uint64_t*>(io_addr),
+        reinterpret_cast<uint64_t*>(VmmManager::MBOX_DMA_SIZE +
+            reinterpret_cast<uint64_t>(io_addr)));
+
+}
+
diff --git a/src/usr/mbox/mbox_dma_buffer.H b/src/usr/mbox/mbox_dma_buffer.H
index 0f39b7c27..534267137 100644
--- a/src/usr/mbox/mbox_dma_buffer.H
+++ b/src/usr/mbox/mbox_dma_buffer.H
@@ -1,26 +1,25 @@
-/*  IBM_PROLOG_BEGIN_TAG
- *  This is an automatically generated prolog.
- *
- *  $Source: src/usr/mbox/mbox_dma_buffer.H $
- *
- *  IBM CONFIDENTIAL
- *
- *  COPYRIGHT International Business Machines Corp. 2012
- *
- *  p1
- *
- *  Object Code Only (OCO) source materials
- *  Licensed Internal Code Source Materials
- *  IBM HostBoot Licensed Internal Code
- *
- *  The source code for this program is not published or other-
- *  wise divested of its trade secrets, irrespective of what has
- *  been deposited with the U.S. Copyright Office.
- *
- *  Origin: 30
- *
- *  IBM_PROLOG_END_TAG
- */
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/mbox/mbox_dma_buffer.H $                              */
+/*                                                                        */
+/* IBM CONFIDENTIAL                                                       */
+/*                                                                        */
+/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/*                                                                        */
+/* p1                                                                     */
+/*                                                                        */
+/* Object Code Only (OCO) source materials                                */
+/* Licensed Internal Code Source Materials                                */
+/* IBM HostBoot Licensed Internal Code                                    */
+/*                                                                        */
+/* The source code for this program is not published or otherwise         */
+/* divested of its trade secrets, irrespective of what has been           */
+/* deposited with the U.S. Copyright Office.                              */
+/*                                                                        */
+/* Origin: 30                                                             */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
 #if !defined(__MBOX_DMA_BUFFER_H)
 #define __MBOX_DMA_BUFFER_H
 
@@ -144,6 +143,12 @@ namespace MBOX
              */
             static uint64_t makeMask(uint64_t i_size);
 
+            /**
+             * Perform VMM operations to allocate physical area in unsecure
+             * region of memory.
+             */
+            void initPhysicalArea(void*& io_addr, uint64_t& o_phys);
+
             enum
             {
                 MAX_MASK_SIZE = sizeof(uint64_t) * 8,
diff --git a/src/usr/targeting/common/xmltohb/simics_MURANO.system.xml b/src/usr/targeting/common/xmltohb/simics_MURANO.system.xml
index e5831d2e8..41214b8e5 100644
--- a/src/usr/targeting/common/xmltohb/simics_MURANO.system.xml
+++ b/src/usr/targeting/common/xmltohb/simics_MURANO.system.xml
@@ -150,10 +150,6 @@
         <default>NONE</default>
     </attribute>
     <attribute>
-        <id>HB_HRMOR_NODAL_BASE</id>
-        <default>0x4000000</default>
-    </attribute>
-    <attribute>
         <id>MSS_MBA_ADDR_INTERLEAVE_BIT</id>
         <default>24</default>
     </attribute>
diff --git a/src/usr/targeting/common/xmltohb/simics_VENICE.system.xml b/src/usr/targeting/common/xmltohb/simics_VENICE.system.xml
index 297b9d718..7028fd31e 100644
--- a/src/usr/targeting/common/xmltohb/simics_VENICE.system.xml
+++ b/src/usr/targeting/common/xmltohb/simics_VENICE.system.xml
@@ -126,10 +126,6 @@
         <default>NONE</default>
     </attribute>
     <attribute>
-        <id>HB_HRMOR_NODAL_BASE</id>
-        <default>0x4000000</default>
-    </attribute>
-    <attribute>
         <id>MSS_MBA_ADDR_INTERLEAVE_BIT</id>
         <default>24</default>
     </attribute>
diff --git a/src/usr/targeting/common/xmltohb/vbu_MURANO.system.xml b/src/usr/targeting/common/xmltohb/vbu_MURANO.system.xml
index ae0d03158..87647971f 100644
--- a/src/usr/targeting/common/xmltohb/vbu_MURANO.system.xml
+++ b/src/usr/targeting/common/xmltohb/vbu_MURANO.system.xml
@@ -142,10 +142,6 @@
         <default>AVP</default>
     </attribute>
     <attribute>
-        <id>HB_HRMOR_NODAL_BASE</id>
-        <default>0x4000000</default>
-    </attribute>
-    <attribute>
         <id>MSS_MBA_ADDR_INTERLEAVE_BIT</id>
         <default>24</default>
     </attribute>
diff --git a/src/usr/targeting/common/xmltohb/vbu_VENICE.system.xml b/src/usr/targeting/common/xmltohb/vbu_VENICE.system.xml
index bf8f14e51..852683420 100644
--- a/src/usr/targeting/common/xmltohb/vbu_VENICE.system.xml
+++ b/src/usr/targeting/common/xmltohb/vbu_VENICE.system.xml
@@ -139,10 +139,6 @@
         <default>AVP</default>
     </attribute>
     <attribute>
-        <id>HB_HRMOR_NODAL_BASE</id>
-        <default>0x4000000</default>
-    </attribute>
-    <attribute>
         <id>MSS_MBA_ADDR_INTERLEAVE_BIT</id>
         <default>24</default>
     </attribute>