Flush/Release/Evict pages functionality

Change-Id: Ic0bb4122164e11f6d13e6850abf8ae9bd32caea2
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/393
Tested-by: Jenkins Server
Reviewed-by: MATTHEW S. BARTH <msbarth@us.ibm.com>

14 files changed, 613 insertions, 121 deletions

diff --git a/src/include/kernel/basesegment.H b/src/include/kernel/basesegment.H
index 1064e1f82..51e99b0bf 100644
--- a/src/include/kernel/basesegment.H
+++ b/src/include/kernel/basesegment.H
@@ -68,7 +68,7 @@ class BaseSegment : public Segment
 
         /**
          * @brief Implementation of the pure-virtual function from Segment.
-         *        Update LRU statistics on the block that owns the address         *
+         *        Update LRU statistics on the block that owns the address
          *
          * @param[in] i_vaddr - Virtual Address of page
          * @param[in] i_stats - Usage statistics
@@ -79,7 +79,8 @@ class BaseSegment : public Segment
         /**
          * @brief Allocates a block of virtual memory of the given size
          * @param i_mq[in] - Message queue to be associated with the block
-         * @param i_va[in] - Base virtual address of the block to be allocated
+         * @param i_va[in] - Page aligned base virtual address of the block
+         *                   to be allocated
          * @param i_size[in] - Requested virtual memory size of the block
          * @return int - 0 for successful block allocation, non-zero otherwise
          */
@@ -92,7 +93,18 @@ class BaseSegment : public Segment
          * @return the physical address bound to the virtual address, or
          *         -EFAULT if i_vaddr not found. @see errno.h
          */
-         virtual uint64_t findPhysicalAddress(uint64_t i_vaddr) const;
+        virtual uint64_t findPhysicalAddress(uint64_t i_vaddr) const;
+
+        /**
+         * @brief Remove pages by a specified operation of the given size
+         * @param[in] i_op - Page removal operation to perform
+         * @param[in] i_vaddr - Virtual address associated to page(s)
+         * @param[in] i_size - Size of memory to perform page removal on
+         * @param[in] i_task - Task requesting page removal.
+         * @return int - 0 for successful page removal, non-zero otherwise
+         */
+        static int mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,
+                void* i_vaddr, uint64_t i_size, task_t* i_task);
 
         /**
          * @brief Sets the page permissions for a given virtual address and size.
@@ -146,6 +158,16 @@ class BaseSegment : public Segment
                                  uint64_t i_size,
                                  uint64_t i_access_type);
 
+        /**
+         * @brief Remove pages by a specified operation of the given size
+         * @param[in] i_op - Page removal operation to perform
+         * @param[in] i_vaddr - Virtual address associated to page(s)
+         * @param[in] i_size - Size of memory to perform page removal on
+         * @param[in] i_task - Task requesting page removal.
+         * @return int - 0 for successful page removal, non-zero otherwise
+         */
+        int _mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op, void* i_vaddr,
+                uint64_t i_size, task_t* i_task);
 
 };
 
diff --git a/src/include/kernel/block.H b/src/include/kernel/block.H
index 405791e94..01187617f 100644
--- a/src/include/kernel/block.H
+++ b/src/include/kernel/block.H
@@ -72,7 +72,7 @@ class Block
               MessageQueue* i_msgQueue = NULL) :
                 iv_baseAddr(i_baseAddr), iv_size(i_size),
                 iv_parent(NULL), iv_nextBlock(NULL), iv_ptes(NULL),
-                iv_msgHdlr(NULL)
+                iv_readMsgHdlr(NULL), iv_writeMsgHdlr(NULL)
             { init(i_msgQueue); };
 
         /**
@@ -117,23 +117,6 @@ class Block
         bool handlePageFault(task_t* i_task, uint64_t i_addr);
 
         /**
-         * @brief Sets the 'present' bit within the Shadow page table
-         *
-         * @param[in] i_vaddr - Virtual address within the Shadow page table
-         */
-        void setIsPresent(void* i_vaddr);
-
-        /**
-         * @brief Adds the page table entry for the given address
-         *
-         * @param[in] i_vaddr - Virtual address to add to the page table
-         *
-         * The permissions set within the Shadow page table are used for
-         * this address
-         */
-        void addPTE(void* i_vaddr);
-
-        /**
          * @brief Locate the physical address of the given virtual address
          *
          * @param[in] i_vaddr virtual address
@@ -171,9 +154,22 @@ class Block
          */
         bool evictPage(ShadowPTE* i_pte);
 
+        /**
+         * @brief Removes a range of pages within a block of virtual memory
+         * @param[in] i_op - Page removal operation to perform
+         * @param[in] i_vaddr - Virtual address associated to page(s)
+         * @param[in] i_size - Size of memory to perform page removal on
+         * @param[in] i_task - Task requesting page removal.
+         * @return int - 0 for successful page removal, non-zero otherwise
+         */
+        int removePages(VmmManager::PAGE_REMOVAL_OPS i_op, void* i_vaddr,
+                        uint64_t i_size, task_t* i_task);
+
         friend class Segment;
         friend class BaseSegment;
         friend class StackSegment;
+        friend class BlockReadMsgHdlr;
+        friend class BlockWriteMsgHdlr;
 
     protected:
         /**
@@ -248,6 +244,30 @@ class Block
          */
         int mmSetPermission(uint64_t i_va, uint64_t i_size, uint64_t i_access_type);
 
+        /**
+         * @brief Adds the page table entry for the given address
+         *
+         * @param[in] i_vaddr - Virtual address to add to the page table
+         *
+         * The permissions set within the Shadow page table are used for
+         * this address
+         */
+        void addPTE(void* i_vaddr);
+
+        /**
+         * @brief Sets the 'present' bit within the Shadow page table
+         *
+         * @param[in] i_vaddr - Virtual address within the Shadow page table
+         */
+        void setIsPresent(void* i_vaddr);
+
+        /**
+         * @brief Effectively removes the given page table entry from the
+         * shadow page table
+         * @param[in] i_pte - Shadow page table entry to release
+         */
+        void releasePTE(ShadowPTE* i_pte);
+
     private:
             /** Base address of the block */
         const uint64_t iv_baseAddr;
@@ -261,8 +281,10 @@ class Block
 
             /** Pointer to the Shadow PTE entries. */
         ShadowPTE* iv_ptes;
-            /** Pointer to the message handler */
-        BlockMsgHdlr* iv_msgHdlr;
+            /** Pointer to message handler(read) */
+        BlockReadMsgHdlr* iv_readMsgHdlr;
+            /** Pointer to message handler(write) */
+        BlockWriteMsgHdlr* iv_writeMsgHdlr;
 
         /**
          * @brief Finish initialization of block.
@@ -286,7 +308,6 @@ class Block
          */
         ShadowPTE* getPTE(uint64_t i_addr) const;
 
-
         Block(const Block&);  // prohibit copy.
         Block& operator=(const Block&);  // prohibit assignment.
 
diff --git a/src/include/kernel/blockmsghdlr.H b/src/include/kernel/blockmsghdlr.H
index 35348a92f..495ab71c3 100644
--- a/src/include/kernel/blockmsghdlr.H
+++ b/src/include/kernel/blockmsghdlr.H
@@ -29,21 +29,44 @@
 #include <stdint.h>
 #include <kernel/types.h>
 #include <kernel/msghandler.H>
+#include <util/locked/list.H>
 
 //Forward declaration.
 class Spinlock;
 class MessageQueue;
 class Block;
 
+struct TaskMsgNode
+{
+    //Next pointer for list.
+    TaskMsgNode* next;
+    //Previous pointer for list.
+    TaskMsgNode* prev;
+
+    task_t* key;
+    uint64_t msgCount;
+};
+
+struct PageAddrNode
+{
+    //Next pointer for list.
+    PageAddrNode* next;
+    //Previous pointer for list.
+    PageAddrNode* prev;
+
+    void* key;
+    uint64_t pageAddr;
+};
+
 /**
- * @class BlockMsgHdlr
- * @brief Class to handle message data for blocks
+ * @class BlockReadMsgHdlr
+ * @brief Class to handle read messages for blocks
  *
  * This class extends from the base MessageHandler so the base send/receive
  * message functions can be utilized. It overrides how to handle the message
  * responses for blocks within the base virtual memory segment.
  */
-class BlockMsgHdlr : public MessageHandler
+class BlockReadMsgHdlr : public MessageHandler
 {
     public:
 
@@ -56,13 +79,14 @@ class BlockMsgHdlr : public MessageHandler
          *                     passed directly onto the MessageHandler
          * @param[in] i_block - Block to associate this message handler to
          */
-        BlockMsgHdlr(Spinlock* i_lock, MessageQueue* i_msgq, Block* i_block) :
-                     MessageHandler(i_lock,i_msgq), iv_block(i_block) {};
+        BlockReadMsgHdlr(Spinlock* i_lock,MessageQueue* i_msgq,
+                         Block* i_block) :
+                         MessageHandler(i_lock,i_msgq), iv_block(i_block) {};
 
         /**
          * @brief Destructor
          */
-        ~BlockMsgHdlr() {};
+        ~BlockReadMsgHdlr() {};
 
         /**
          * @brief Handle response to 'send message' associated with this block
@@ -84,4 +108,72 @@ class BlockMsgHdlr : public MessageHandler
         Block* const iv_block;
 };
 
+/**
+ * @class BlockWriteMsgHdlr
+ * @brief Class to handle write messages for blocks
+ *
+ * This class extends from the base MessageHandler so the base send/receive
+ * message functions can be utilized. It overrides how to handle the message
+ * responses for blocks within the base virtual memory segment.
+ */
+class BlockWriteMsgHdlr : public MessageHandler
+{
+    public:
+
+        /**
+         * @brief Constructor
+         *
+         * @param[in] i_lock - Subsystem lock for this message handler, passed
+         *                     directly onto the MessageHandler
+         * @param[in] i_msgq - Queue used to send messages into userspace,
+         *                     passed directly onto the MessageHandler
+         * @param[in] i_block - Block to associate this message handler to
+         */
+        BlockWriteMsgHdlr(Spinlock* i_lock,MessageQueue* i_msgq,
+                          Block* i_block) :
+                          MessageHandler(i_lock,i_msgq), iv_block(i_block) {};
+
+        /**
+         * @brief Destructor
+         */
+        ~BlockWriteMsgHdlr() {};
+
+        /**
+         * @brief Handle response to 'send message' associated with this block
+         *
+         *  @param[in] i_type - The message type previously sent.
+         *  @param[in] i_key - The key value for the received message.
+         *  @param[in] i_task - The deferred task.
+         *  @param[in] i_rc - The response rc from userspace.
+         *
+         *  @return HandleResult - The desired behavior on the 'recv message'
+         *                         interface for this <key, task> pair.
+         */
+        virtual HandleResult handleResponse(msg_sys_types_t i_type,void* i_key,
+                                            task_t* i_task,int i_rc);
+
+        /**
+         * @brief Increments the number of messages sent from the given task
+         * @param[in] i_task - Associated task to message sent
+         */
+        void incMsgCount(task_t* i_task);
+
+        /**
+         * @brief Adds the virtual address to page address association for
+         * page removal upon response
+         * @param[in] i_vaddr - Virtual address sent on message
+         * @param[in] i_pgAddr - Page address to be removed
+         */
+        void addVirtAddr(void* i_vaddr,uint64_t i_pgAddr);
+
+    private:
+
+        /* Associated block for this message handler */
+        Block* const iv_block;
+        /* List of associated tasks to number of messages */
+        Util::Locked::List<TaskMsgNode, task_t*> iv_msgGrpList;
+        /* List of associated virtual address to page address */
+        Util::Locked::List<PageAddrNode, void*> iv_va2paList;
+};
+
 #endif
diff --git a/src/include/kernel/vmmmgr.H b/src/include/kernel/vmmmgr.H
index ce7aedafb..864eed0fd 100644
--- a/src/include/kernel/vmmmgr.H
+++ b/src/include/kernel/vmmmgr.H
@@ -61,6 +61,23 @@ class VmmManager
             CRITICAL,
         };
 
+        /**
+         * Kernel mapped page removal operations
+         *
+         * RELEASE : Writes dirty&write-tracked pages out to a storage device
+         *           and removes other pages
+         * FLUSH   : Only writes dirty&write-tracked pages out to a storage
+         *           device
+         * EVICT   : (Kernel) Writes dirty&write-tracked pages out to a storage
+         *           device and removes other pages
+         */
+        enum PAGE_REMOVAL_OPS
+        {
+            RELEASE = 0,
+            FLUSH = 1,
+            EVICT = 2,
+        };
+
         static void init();
         static void init_slb();
 
@@ -98,7 +115,8 @@ class VmmManager
         /**
          * @brief Allocates a block of virtual memory of the given size
          * @param i_mq[in] - Message queue to be associated with the block
-         * @param i_va[in] - Base virtual address of the block to be allocated
+         * @param i_va[in] - Page aligned base virtual address of the block
+         *                   to be allocated
          * @param i_size[in] - Requested virtual memory size of the block
          * @return int - 0 for successful block allocation, non-zero otherwise
          */
@@ -127,14 +145,19 @@ class VmmManager
          * @param[in] i_op - Page removal operation to perform
          * @param[in] i_vaddr - Virtual address associated to page(s)
          * @param[in] i_size - Size of memory to perform page removal on
+         * @param[in] i_task - OPTIONAL:Task requesting page removal.
          * @return int - 0 for successful page removal, non-zero otherwise
          *
          * The given virtual address will be 'rounded' down to the nearest page
          * boundary, along with the given size will be 'rounded' up to the
          * nearest divisible page size.
+         *
+         * When a task is given, it will be deferred until all pages requested
+         * for removal have completed.
          */
-        static int mmRemovePages(PAGE_REMOVAL_OPS i_op, void* i_vaddr,
-                                 uint64_t i_size);
+        static int mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,
+                                 void* i_vaddr, uint64_t i_size,
+                                 task_t* i_task = NULL);
         /**
          * @brief Sets the permissions for a given page or range of pages
          * @param i_va[in] - Virtual address of the page to update permission
@@ -180,6 +203,13 @@ class VmmManager
         /** See flushPageTable */
         void _flushPageTable( void );
 
+        /** See mmAllocBlock */
+        int _mmAllocBlock(MessageQueue* i_mq,void* i_va,uint64_t i_size);
+
+        /** See mmRemovePages */
+        int _mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,void* i_vaddr,
+                           uint64_t i_size,task_t* i_task);
+
     public:
         friend class Block;
 
diff --git a/src/include/sys/mm.h b/src/include/sys/mm.h
index bfd5a1640..3064e371a 100644
--- a/src/include/sys/mm.h
+++ b/src/include/sys/mm.h
@@ -34,11 +34,16 @@ extern "C"
 
 /**
  * Page removal operations
+ *
+ * RELEASE : Writes dirty&write-tracked pages out to a storage device
+ *           and removes other pages
+ * FLUSH   : Only writes dirty&write-tracked pages out to a storage
+ *           device
  */
 enum PAGE_REMOVAL_OPS
 {
-    RELEASE,
-    FLUSH,
+    RELEASE = 0,
+    FLUSH = 1,
 };
 
 /**
@@ -60,7 +65,8 @@ enum PAGE_PERMISSIONS
  *  @brief System call to allocate virtual memory block in the base segment
  *
  *  @param[in] mq - Message queue to be associated with the block
- *  @param[in] va - Base virtual address of the block to be allocated
+ *  @param[in] va - Page aligned base virtual address of the block
+ *                  to be allocated
  *  @param[in] size - Requested virtual memory size of the block
  *
  *  @return int - 0 for successful block allocation, non-zero otherwise
diff --git a/src/include/usr/vmmconst.h b/src/include/usr/vmmconst.h
index 2eb1ea578..ac5ee40be 100644
--- a/src/include/usr/vmmconst.h
+++ b/src/include/usr/vmmconst.h
@@ -67,4 +67,14 @@
 #define SLBE_b 12
 
 
+/**
+ * Test Constants
+ */
+/** Base virtual address used in remove pages test */
+#define VMM_VADDR_RMVPAGE_TEST (700 * GIGABYTE);
+
+/** Block size used in remove pages test */
+#define VMM_SIZE_RMVPAGE_TEST (8 * PAGESIZE);
+
+
 #endif /* _VMMCONST_H */
diff --git a/src/kernel/basesegment.C b/src/kernel/basesegment.C
index 8eb7e5484..4cb2c39cb 100644
--- a/src/kernel/basesegment.C
+++ b/src/kernel/basesegment.C
@@ -118,7 +118,8 @@ int BaseSegment::_mmAllocBlock(MessageQueue* i_mq,void* i_va,uint64_t i_size)
     //Verify input address and size falls within this segment's address range
     if (l_vaddr < this->getBaseAddress() ||
         l_vaddr >= (this->getBaseAddress() + (1ull << SLBE_s)) ||
-        (l_blockSizeTotal + ALIGN_PAGE(i_size)) >= (1ull << SLBE_s))
+        (l_blockSizeTotal + ALIGN_PAGE(i_size)) >= (1ull << SLBE_s) ||
+        (l_vaddr != ALIGN_PAGE_DOWN(l_vaddr)))
     {
         return -EINVAL;
     }
@@ -198,3 +199,25 @@ void BaseSegment::castOutPages(uint64_t i_type)
     // Could try again with a more agressive constraint if cast_out == 0 ????
     if(cast_out) printkd("Cast out %ld pages,Type=%ld\n",cast_out,i_type);
 }
+/**
+ * STATIC
+ * Remove pages by a specified operation of the given size
+ */
+int BaseSegment::mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,
+                               void* i_vaddr, uint64_t i_size, task_t* i_task)
+{
+    return Singleton<BaseSegment>::instance()._mmRemovePages(i_op,i_vaddr,
+                                                             i_size,i_task);
+}
+
+/**
+ * Remove pages by a specified operation of the given size
+ */
+int BaseSegment::_mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,
+                                void* i_vaddr, uint64_t i_size, task_t* i_task)
+{
+    //Don't allow removal of pages for base block
+    return (iv_block->iv_nextBlock ?
+            iv_block->iv_nextBlock->removePages(i_op,i_vaddr,i_size,i_task):
+            -EINVAL);
+}
diff --git a/src/kernel/block.C b/src/kernel/block.C
index f72a8f6f7..5b1520b1b 100644
--- a/src/kernel/block.C
+++ b/src/kernel/block.C
@@ -27,6 +27,8 @@
 
 #include <sys/msg.h>
 
+#include <util/align.H>
+
 #include <kernel/block.H>
 #include <kernel/spte.H>
 #include <kernel/vmmmgr.H>
@@ -39,6 +41,9 @@ Block::~Block()
 {
     // Release shadow PTE array.
     delete[] iv_ptes;
+    //Release message handlers
+    delete iv_readMsgHdlr;
+    delete iv_writeMsgHdlr;
 
     // Delete next block in the chain.
     if (iv_nextBlock)
@@ -51,11 +56,14 @@ void Block::init(MessageQueue* i_msgQ)
 {
     // Create a shadow PTE for each page.
     iv_ptes = new ShadowPTE[iv_size / PAGESIZE];
-    this->iv_msgHdlr = NULL;
     if (i_msgQ != NULL)
     {
-        //Create message handler attribute for this block with this msgq
-        this->iv_msgHdlr = new BlockMsgHdlr(VmmManager::getLock(),i_msgQ,this);
+        //Create message handler to handle read operations for this block
+        this->iv_readMsgHdlr =
+                new BlockReadMsgHdlr(VmmManager::getLock(),i_msgQ,this);
+        //Create message handler to handle write operations for this block
+        this->iv_writeMsgHdlr =
+                new BlockWriteMsgHdlr(VmmManager::getLock(),i_msgQ,this);
     }
 }
 
@@ -80,7 +88,7 @@ bool Block::handlePageFault(task_t* i_task, uint64_t i_addr)
 
     if (!pte->isPresent())
     {
-        if (this->iv_msgHdlr != NULL)
+        if (this->iv_readMsgHdlr != NULL)
         {
             void* l_page = reinterpret_cast<void*>(pte->getPageAddr());
             //If the page data is zero, create the page
@@ -94,7 +102,7 @@ bool Block::handlePageFault(task_t* i_task, uint64_t i_addr)
                 pte->setWritable(true);
             }
             //Send message to handler to read page
-            this->iv_msgHdlr->sendMessage(MSG_MM_RP_READ,
+            this->iv_readMsgHdlr->sendMessage(MSG_MM_RP_READ,
                     reinterpret_cast<void*>(l_addr_palign),l_page,i_task);
             //Done(waiting for response)
             return true;
@@ -406,7 +414,6 @@ size_t Block::castOutPages(uint64_t i_type)
     return cast_out;
 }
 
-
 int Block::mmSetPermission(uint64_t i_va, uint64_t i_size,uint64_t i_access_type)
 {
  int l_rc = 0;
@@ -508,5 +515,75 @@ int Block::mmSetPermission(uint64_t i_va, uint64_t i_size,uint64_t i_access_type
 
  }
  return l_rc;
+}
+
+int Block::removePages(VmmManager::PAGE_REMOVAL_OPS i_op, void* i_vaddr,
+                                 uint64_t i_size, task_t* i_task)
+{
+    uint64_t l_vaddr = reinterpret_cast<uint64_t>(i_vaddr);
+    //Align virtual address & size to page boundary
+    /*The given virtual address will be 'rounded' down to the nearest page
+      boundary, along with the given size will be 'rounded' up to the
+      nearest divisible page size.*/
+    uint64_t l_aligned_va = ALIGN_PAGE_DOWN(l_vaddr);
+    uint64_t l_aligned_size = ALIGN_PAGE(i_size);
+    //Find block containing virtual address
+    if(!this->isContained(l_aligned_va))
+    {
+        return (iv_nextBlock ?
+                iv_nextBlock->removePages(i_op,i_vaddr,i_size,i_task):-EINVAL);
+    }
+    else if ((l_aligned_va+l_aligned_size) > (this->iv_baseAddr+this->iv_size))
+    {
+        return -EINVAL;
+    }
 
+    //Perform requested page removal operation
+    for (l_vaddr = l_aligned_va;l_vaddr < (l_aligned_va+l_aligned_size);
+         l_vaddr+= PAGESIZE)
+    {
+        ShadowPTE* pte = getPTE(l_vaddr);
+        uint64_t pageAddr = pte->getPageAddr();
+        if (pte->isPresent() && (0 != pageAddr))
+        {
+            //Delete from HW page table immediately
+            PageTableManager::delEntry(l_vaddr);
+            if (pte->isDirty() && pte->isWriteTracked() &&
+                this->iv_writeMsgHdlr != NULL)
+            {
+                releasePTE(pte);
+                //Send write msg with the page address
+                if (i_task != NULL)
+                {
+                    this->iv_writeMsgHdlr->incMsgCount(i_task);
+                }
+                this->iv_writeMsgHdlr->addVirtAddr(
+                        reinterpret_cast<void*>(l_vaddr),pageAddr);
+                this->iv_writeMsgHdlr->sendMessage(MSG_MM_RP_WRITE,
+                        reinterpret_cast<void*>(l_vaddr),
+                        reinterpret_cast<void*>(pageAddr),i_task);
+            }
+            else if (pte->isDirty() && !pte->isWriteTracked() &&
+                     i_op == VmmManager::EVICT)
+            {
+                //Leave as 'printk' to note page was skipped
+                printk("Block::removePages >> Unable to EVICT ");
+                printk("dirty page thats not write tracked: ");
+                printk("va: 0x%.16lX, pa: 0x%.16lX \n",l_vaddr, pageAddr);
+            }
+            else if (i_op != VmmManager::FLUSH)
+            {
+                //'Release' page entry
+                releasePTE(pte);
+                PageManager::freePage(reinterpret_cast<void*>(pageAddr));
+            }
+        }
+    }
+    return 0;
+}
+
+void Block::releasePTE(ShadowPTE* i_pte)
+{
+    i_pte->setPresent(false);
+    i_pte->setPageAddr(NULL);
 }
diff --git a/src/kernel/blockmsghdlr.C b/src/kernel/blockmsghdlr.C
index cf1683cab..fc49ae89d 100644
--- a/src/kernel/blockmsghdlr.C
+++ b/src/kernel/blockmsghdlr.C
@@ -22,8 +22,9 @@
 //  IBM_PROLOG_END
 #include <kernel/blockmsghdlr.H>
 #include <kernel/block.H>
+#include <kernel/pagemgr.H>
 
-MessageHandler::HandleResult BlockMsgHdlr::handleResponse(
+MessageHandler::HandleResult BlockReadMsgHdlr::handleResponse(
         msg_sys_types_t i_type, void* i_key, task_t* i_task, int i_rc)
 {
     if (i_rc != 0)
@@ -41,3 +42,73 @@ MessageHandler::HandleResult BlockMsgHdlr::handleResponse(
         return SUCCESS;
     }
 }
+
+MessageHandler::HandleResult BlockWriteMsgHdlr::handleResponse(
+        msg_sys_types_t i_type, void* i_key, task_t* i_task, int i_rc)
+{
+    //Default to indicate nothing specific has been done for this response.
+    MessageHandler::HandleResult l_result = UNHANDLED_RC;
+    if (i_rc != 0)
+    {
+        //Request default behavior of resume/kill task based on rc.
+        return l_result;
+    }
+    //Find the virtual address to page address mapping to know which page
+    //address to release since only the virtual address is available on the msg
+    PageAddrNode* l_paNode = iv_va2paList.find(i_key);
+    if (l_paNode)
+    {
+        l_result = SUCCESS;
+        //Complete page removal and remove list entry
+        PageManager::freePage(reinterpret_cast<void*>(l_paNode->pageAddr));
+        iv_va2paList.erase(l_paNode);
+        delete l_paNode;
+    }
+    //Not handling a reponse from kernel
+    if (i_task != NULL)
+    {
+        //Find the task's msg count to know how many messages were sent
+        //to remove pages and whether to continue deferring the task or not
+        TaskMsgNode* l_tmNode = iv_msgGrpList.find(i_task);
+        if (l_tmNode)
+        {
+            //Last message for the given task
+            if (l_tmNode->msgCount == 1 &&
+                (l_tmNode->next == NULL && l_tmNode->prev == NULL))
+            {
+                l_result = SUCCESS;
+                iv_msgGrpList.erase(l_tmNode);
+                delete l_tmNode;
+            }
+            else
+            {
+                l_result = CONTINUE_DEFER;
+                l_tmNode->msgCount--;
+            }
+        }
+    }
+
+    return l_result;
+}
+
+void BlockWriteMsgHdlr::incMsgCount(task_t* i_task)
+{
+    TaskMsgNode* l_tmNode = iv_msgGrpList.find(i_task);
+    if (l_tmNode == NULL)
+    {
+        //Add task to list and set message count to 1
+        l_tmNode = new TaskMsgNode();
+        l_tmNode->key = i_task;
+        l_tmNode->msgCount = 0;
+        iv_msgGrpList.insert(l_tmNode);
+    }
+    l_tmNode->msgCount++;
+}
+
+void BlockWriteMsgHdlr::addVirtAddr(void* i_vaddr,uint64_t i_pgAddr)
+{
+    PageAddrNode* l_paNode = new PageAddrNode();
+    l_paNode->key = i_vaddr;
+    l_paNode->pageAddr = i_pgAddr;
+    iv_va2paList.insert(l_paNode);
+}
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index ecfdd9b1b..cb4650c7a 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -516,11 +516,12 @@ namespace Systemcalls
      */
     void MmRemovePages(task_t* t)
     {
-        PAGE_REMOVAL_OPS oper = (PAGE_REMOVAL_OPS)TASK_GETARG0(t);
+        VmmManager::PAGE_REMOVAL_OPS oper =
+                (VmmManager::PAGE_REMOVAL_OPS)TASK_GETARG0(t);
         void* vaddr = (void*)TASK_GETARG1(t);
         uint64_t size = (uint64_t)TASK_GETARG2(t);
 
-        TASK_SETRTN(t, VmmManager::mmRemovePages(oper,vaddr,size));
+        TASK_SETRTN(t, VmmManager::mmRemovePages(oper,vaddr,size,t));
     }
 
      /**
diff --git a/src/kernel/vmmmgr.C b/src/kernel/vmmmgr.C
index c923d80e0..cfa29b569 100644
--- a/src/kernel/vmmmgr.C
+++ b/src/kernel/vmmmgr.C
@@ -147,9 +147,16 @@ bool VmmManager::_pteMiss(task_t* t, uint64_t effAddr)
 
 int VmmManager::mmAllocBlock(MessageQueue* i_mq,void* i_va,uint64_t i_size)
 {
-    return BaseSegment::mmAllocBlock(i_mq,i_va,i_size);
+    return Singleton<VmmManager>::instance()._mmAllocBlock(i_mq,i_va,i_size);
 }
 
+int VmmManager::_mmAllocBlock(MessageQueue* i_mq,void* i_va,uint64_t i_size)
+{
+    lock.lock();
+    int rc = BaseSegment::mmAllocBlock(i_mq,i_va,i_size);
+    lock.unlock();
+    return rc;
+}
 
 Spinlock* VmmManager::getLock()
 {
@@ -169,12 +176,21 @@ uint64_t VmmManager::_findPhysicalAddress(uint64_t i_vaddr)
     return paddr;
 }
 
-int VmmManager::mmRemovePages(PAGE_REMOVAL_OPS i_op, void* i_vaddr,
-                              uint64_t i_size)
+int VmmManager::mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op, void* i_vaddr,
+                              uint64_t i_size, task_t* i_task)
 {
-    return 0;
+    return Singleton<VmmManager>::instance()._mmRemovePages(i_op,i_vaddr,
+                                                            i_size,i_task);
 }
 
+int VmmManager::_mmRemovePages(VmmManager::PAGE_REMOVAL_OPS i_op,void* i_vaddr,
+                               uint64_t i_size,task_t* i_task)
+{
+    lock.lock();
+    int rc = BaseSegment::mmRemovePages(i_op,i_vaddr,i_size,i_task);
+    lock.unlock();
+    return rc;
+}
 
 int VmmManager::mmSetPermission(void* i_va, uint64_t i_size, uint64_t i_access_type)
 {
diff --git a/src/usr/testcore/kernel/ptmgrtest.H b/src/usr/testcore/kernel/ptmgrtest.H
index c32573a43..1d87b312e 100644
--- a/src/usr/testcore/kernel/ptmgrtest.H
+++ b/src/usr/testcore/kernel/ptmgrtest.H
@@ -96,7 +96,7 @@ class ptmgrtest : public CxxTest::TestSuite
     void test_hash40( void )
     {
         TS_TRACE( ">> ptmgrtest::test_hash40 <<" );
-        uint64_t fails, total = 0;
+        uint64_t fails = 0, total = 0;
 
         // Initialize the Page Table
         PageTableManager* ptmgr = new PageTableManager(true);
diff --git a/src/usr/testcore/kernel/slbtest.H b/src/usr/testcore/kernel/segmenttest.H
index fef88b5d5..d98e3c7b1 100644
--- a/src/usr/testcore/kernel/slbtest.H
+++ b/src/usr/testcore/kernel/segmenttest.H
@@ -1,7 +1,7 @@
 //  IBM_PROLOG_BEGIN_TAG
 //  This is an automatically generated prolog.
 //
-//  $Source: src/usr/testcore/kernel/slbtest.H $
+//  $Source: src/usr/testcore/kernel/segmenttest.H $
 //
 //  IBM CONFIDENTIAL
 //
@@ -20,26 +20,24 @@
 //  Origin: 30
 //
 //  IBM_PROLOG_END
-#ifndef __SLBTEST_H
-#define __SLBTEST_H
+#ifndef __SEGMENTTEST_H
+#define __SEGMENTTEST_H
 /**
- *  @file slbtest.H
+ *  @file segmenttest.H
  *
- *  @brief Test cases for the segment lookaside buffers
+ *  @brief Test cases for the virtual memory segments
 */
 #include <cxxtest/TestSuite.H>
 #include <arch/ppc.H>
 #include <sys/time.h>
 #include <sys/task.h>
 #include <sys/mmio.h>
-#include <sys/mm.h>
+#include <usr/vmmconst.h>
 
-class slbtest: public CxxTest::TestSuite
+class segmenttest: public CxxTest::TestSuite
 {
     public:
 
-        static volatile int rc;
-
         void testDevSeg()
         {
             int rc = 0;
@@ -96,67 +94,6 @@ class slbtest: public CxxTest::TestSuite
             }
         }
 
-        void testPageRemoval()
-        {
-            int rc = -1;
-            uint64_t va = 0xC800000000; //800GB
-            uint64_t size = 0x100000; //1MB
-            rc = mm_remove_pages(FLUSH,reinterpret_cast<void*>(va),size);
-            if (rc != 0)
-            {
-                TS_FAIL("Failed to remove pages\n");
-	    }
-	}
-
-        void testSetPerm()
-        {
-            int rc = 1;
-            uint64_t va = 0xC800000000; //800GB
-            uint64_t size = 0x0;
-            uint64_t access = (uint64_t)(WRITABLE | ALLOCATE_FROM_ZERO); //Access value
-            printkd("Update Page Permissions. Writable/Allocate from zero to addr 800Gb and size = 0 \n");
-            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
-            if (rc != 0)
-            {
-                TS_FAIL(" 1   Failed to Update permissions.\n");
-            }
-
-            rc = 1;
-            size = PAGESIZE * 3;
-            access = (uint64_t)(WRITE_TRACKED); //Access value
-            printkd("Update Page Permissions. write_tracked to addr 800Gb and size = 3 pages\n");
-            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
-            if (rc != 0)
-            {
-                TS_FAIL("2    Failed to Update permissions.\n");
-            }
-            rc = 1;
-            va = 0xC800000000 + (PAGESIZE * 10);
-            size = PAGESIZE * 2;
-            access = (uint64_t)(EXECUTABLE); //Access value
-            printkd("Update Page Permissions. executable to addr C800A000 and size = 2 pages\n");
-            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
-            if (rc != 0)
-            {
-                TS_FAIL("3   Failed to Update permissions.\n");
-            }
-            rc = 0;
-            va = 0xC800000000; //800GB
-            size = 0x0;
-            access = (uint64_t)(WRITABLE | EXECUTABLE); //Access value
-            printkd("Update Page Permissions. Writable/executable to addr 800Gb and size = 0 \n");
-            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
-            if (rc == 0)
-            {
-                printk("Error .. invalid combination that did not get detected\n");
-                TS_FAIL(" 4   Failed to detect a bad parm condition.\n");
-            }
-
-	}
-
-    private:
-
 };
-volatile int slbtest::rc = 0;
 
 #endif
diff --git a/src/usr/testcore/kernel/vmmpagetest.H b/src/usr/testcore/kernel/vmmpagetest.H
new file mode 100644
index 000000000..39090b542
--- /dev/null
+++ b/src/usr/testcore/kernel/vmmpagetest.H
@@ -0,0 +1,186 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/usr/testcore/kernel/vmmpagetest.H $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2011
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+#ifndef __VMMPAGETEST_H
+#define __VMMPAGETEST_H
+/**
+ *  @file vmmpagetest.H
+ *
+ *  @brief Test cases for handling pages within virtual memory
+*/
+#include <cxxtest/TestSuite.H>
+#include <arch/ppc.H>
+#include <sys/time.h>
+#include <sys/task.h>
+#include <sys/mm.h>
+#include <usr/vmmconst.h>
+
+class vmmpagetest: public CxxTest::TestSuite
+{
+    public:
+
+        static volatile int rc;
+        //Testing page removal variables
+        static msg_q_t iv_mq;
+        static uint64_t iv_va;
+        static uint64_t iv_size;
+
+        void testPageSetup()
+        {
+            uint64_t initPerm = (uint64_t)(READ_ONLY);
+            rc = mm_alloc_block(iv_mq,reinterpret_cast<void*>(iv_va),iv_size);
+            if (rc != 0)
+            {
+                TS_FAIL("Unable to allocate block.\n");
+            }
+            rc = mm_set_permission(reinterpret_cast<void*>(iv_va),iv_size,
+                                   initPerm);
+            if (rc != 0)
+            {
+                TS_FAIL("Failed to set block permissions to READ_ONLY.\n");
+            }
+            task_create(testDaemon, NULL);
+        }
+
+        void testReadPageRelease()
+        {
+            //Dependent on block being initialized to READ_ONLY
+            (*(volatile uint64_t *)(iv_va+2*PAGESIZE)) = 0x11111111; sync();
+            rc = mm_remove_pages(RELEASE,
+                    reinterpret_cast<void*>(iv_va),iv_size);
+            if (rc != 0)
+            {
+                TS_FAIL("Failed to release read pages\n");
+            }
+        }
+
+        void testWriteTrackPageFlush()
+        {
+            (*(volatile uint64_t *)iv_va) = 0x12345678; sync();
+            (*(volatile uint64_t *)(iv_va+PAGESIZE)) = 0x87654321; sync();
+            (*(volatile uint64_t *)(iv_va+2*PAGESIZE)) = 0x22222222; sync();
+            uint64_t updPerm = (uint64_t)(WRITE_TRACKED);
+            rc = mm_set_permission(reinterpret_cast<void*>(iv_va),0,updPerm);
+            if (rc != 0)
+            {
+                TS_FAIL(
+                "Failed to set WRITE_TRACKED permissions on first page.\n");
+            }
+            rc = mm_set_permission(reinterpret_cast<void*>((iv_va+PAGESIZE)),0,
+                                   updPerm);
+            if (rc != 0)
+            {
+                TS_FAIL(
+                "Failed to set WRITE_TRACKED permissions on second page.\n");
+            }
+            rc = mm_remove_pages(FLUSH,
+                                 reinterpret_cast<void*>(iv_va),iv_size);
+            if (rc != 0)
+            {
+                TS_FAIL("Failed to flush write tracked pages\n");
+            }
+        }
+
+        void testWriteTrackPageRelease()
+        {
+            (*(volatile uint64_t *)(iv_va+2*PAGESIZE)) = 0x33333333; sync();
+            (*(volatile uint64_t *)iv_va) = 0x12121212; sync();
+            rc = mm_remove_pages(RELEASE,
+                    reinterpret_cast<void*>(iv_va),iv_size);
+            if (rc != 0)
+            {
+                TS_FAIL("Failed to release write track pages\n");
+            }
+        }
+
+        void testSetPerm()
+        {
+            int rc = 1;
+            uint64_t va = 0xC800000000; //800GB
+            uint64_t size = 0x0;
+            uint64_t access = (uint64_t)(WRITABLE | ALLOCATE_FROM_ZERO); //Access value
+            printkd("Update Page Permissions. Writable/Allocate from zero to addr 800Gb and size = 0 \n");
+            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
+            if (rc != 0)
+            {
+                TS_FAIL(" 1   Failed to Update permissions.\n");
+            }
+
+            rc = 1;
+            size = PAGESIZE * 3;
+            access = (uint64_t)(WRITE_TRACKED); //Access value
+            printkd("Update Page Permissions. write_tracked to addr 800Gb and size = 3 pages\n");
+            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
+            if (rc != 0)
+            {
+                TS_FAIL("2    Failed to Update permissions.\n");
+            }
+            rc = 1;
+            va = 0xC800000000 + (PAGESIZE * 10);
+            size = PAGESIZE * 2;
+            access = (uint64_t)(EXECUTABLE); //Access value
+            printkd("Update Page Permissions. executable to addr C800A000 and size = 2 pages\n");
+            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
+            if (rc != 0)
+            {
+                TS_FAIL("3   Failed to Update permissions.\n");
+            }
+            rc = 0;
+            va = 0xC800000000; //800GB
+            size = 0x0;
+            access = (uint64_t)(WRITABLE | EXECUTABLE); //Access value
+            printkd("Update Page Permissions. Writable/executable to addr 800Gb and size = 0 \n");
+            rc = mm_set_permission(reinterpret_cast<void*>(va), size, access);
+            if (rc == 0)
+            {
+                printk("Error .. invalid combination that did not get detected\n");
+                TS_FAIL(" 4   Failed to detect a bad parm condition.\n");
+            }
+        }
+
+    private:
+
+        static void testDaemon(void* unused)
+        {
+            msg_t* message = NULL;
+            uint64_t ea = 0;
+            while (1)
+            {
+                message = msg_wait(iv_mq);
+                if (message)
+                {
+                    ea = message->data[0];
+                    printkd("Effective Addr: 0x%lX, %s\n",ea,
+                            message->type==MSG_MM_RP_READ?"READ":"WRITE");
+                    message->data[1] = 0;
+                    rc = msg_respond(iv_mq, message);
+                }
+            }
+        }
+
+};
+volatile int vmmpagetest::rc = 0;
+msg_q_t vmmpagetest::iv_mq = msg_q_create();
+uint64_t vmmpagetest::iv_va = VMM_VADDR_RMVPAGE_TEST;
+uint64_t vmmpagetest::iv_size = VMM_SIZE_RMVPAGE_TEST;
+
+#endif