From 2583b46750d1e1c062763eff204baa1460e7ae90 Mon Sep 17 00:00:00 2001 From: Dan Crowell Date: Wed, 6 Jul 2011 10:16:21 -0500 Subject: Initial checkin of PageTableManager - RTC:3195 Add new delRange method to delete based on page numbers, part of RTC:3195 Plus code review comments from previous commit - http://gfw160.austin.ibm.com:8080/gerrit/188 Change-Id: Ie45365162cf1367c5c0dcc3afc2907a6ddfa53d3 Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/188 Tested-by: Jenkins Server Reviewed-by: A. Patrick Williams III Reviewed-by: MATTHEW S. BARTH --- src/include/kernel/ptmgr.H | 406 ++++++++++++++++ src/include/kernel/vmmmgr.H | 96 +--- src/kernel/makefile | 2 +- src/kernel/ptmgr.C | 936 ++++++++++++++++++++++++++++++++++++ src/kernel/vmmmgr.C | 43 +- src/makefile | 3 +- src/usr/testcore/kernel/ptmgrtest.H | 392 +++++++++++++++ 7 files changed, 1750 insertions(+), 128 deletions(-) create mode 100644 src/include/kernel/ptmgr.H create mode 100644 src/kernel/ptmgr.C create mode 100644 src/usr/testcore/kernel/ptmgrtest.H diff --git a/src/include/kernel/ptmgr.H b/src/include/kernel/ptmgr.H new file mode 100644 index 000000000..db552ada4 --- /dev/null +++ b/src/include/kernel/ptmgr.H @@ -0,0 +1,406 @@ +#ifndef __KERNEL_PTMGR_H +#define __KERNEL_PTMGR_H + +#include +#include +#include + +/** + * @class PageTableManager + * @brief Manages the Page Table in the hardware + */ +class PageTableManager +{ + public: + // Public Constants + + /** + * Status Values + */ + enum + { + PTE_UNKNOWN = 0x0000000000000000, /**< Entry wasn't found */ + PTE_PRESENT = 0x0000000000000001, /**< Entry is present in table */ + PTE_VALID = 0x0000000000000002, /**< Entry is valid */ + PTE_READ_ONLY = 0x0000000000000004, /**< Read-Only */ + PTE_EXECUTE = 0x0000000000000008, /**< Execute permission */ + PTE_CACHE_INHIBITED = 0x0000000000000010, /**< Cache-Inhibited Access */ + PTE_MODIFIED = 0x0000000000000020, /**< Page has been modified */ + PTE_ACCESSED = 0x0000000000000040, /**< Page has been accessed */ + }; + + /** + * Page Table Constants + */ + enum + { + PT_SIZE = (1 << 18), /**< Size of Page Table in bytes */ + PTE_SIZE = 2*sizeof(uint64_t), /**< Size of a single Page Table Entry in bytes */ + PTEG_SIZE = 8, /**< Number of PTEs in a single PTE Group */ + PTEG_COUNT = (PT_SIZE / PTE_SIZE) / PTEG_SIZE, /**< Number of PTEGs in the Page Table */ + INVALID_PN = 0xFFFFFFFFFFFFFFFF, /**< Invalid value for a Page Number return */ + }; + + /** + * @brief Static Initializer + */ + static void init(); + + /** + * @brief Add an entry to the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be mapped (full address) + * @param[in] i_page Real physical page number to map to (page number) + * @param[in] i_accessType Type of access page will be given + */ + static void addEntry( uint64_t i_vAddr, + uint64_t i_page, + VmmManager::ACCESS_TYPES i_accessType ); + + /** + * @brief Remove an entry from the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be removed (full address) + */ + static void delEntry( uint64_t i_vAddr ); + + /** + * @brief Remove a range of entries from the hardware page table + * + * @param[in] i_vAddrStart Beginning of VA range to remove (full address) + * @param[in] i_vAddrFinish End of VA range to remove (full address) + */ + static void delRangeVA( uint64_t i_vAddrStart, + uint64_t i_vAddrFinish ); + + + /** + * @brief Remove a range of entries from the hardware page table + * + * @param[in] i_pnStart First Physical Page to remove (page number) + * @param[in] i_pnFinish Last Physical Page to remove (page number) + */ + static void delRangePN( uint64_t i_pnStart, + uint64_t i_pnFinish ); + + /** + * @brief Return status information about an entry in the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be queried (full address) + * @param[out] o_pn Real physical page number to map to, INVALID_PN if PTE is invalid + * + * @return uint64_t ORed combination of status flags + */ + static uint64_t getStatus( uint64_t i_vAddr, + uint64_t& o_pn ); + + /** + * @brief Print out the contents of a PTE to the printk buffer + * + * @param[in] i_label string to display along with the PTE data + * @param[in] i_pteAddr address/pointer to the PTE to display + * @param[in] i_verbose true=break out all attributes, false=abbreviated output + */ + static void printPTE( const char* i_label, + uint64_t i_pteAddr, + bool i_verbose = false ); + + /** + * @brief Print out the contents of a PTE based on a VA + * + * @param[in] i_va Virtual address that is part of the page the PTE points to + * @param[in] i_verbose true=break out all attributes, false=abbreviated output + */ + static void printPTE( uint64_t i_va, + bool i_verbose = false ); + + /** + * @brief Print out the entire Page Table + */ + static void printPT( void ); + + + protected: + /** + * @brief Constructor + * + * @param[in] i_userSpace true=declare a local page table for user-space testing + */ + PageTableManager( bool i_userSpace = false ); + + /** + * @brief Destructor + */ + ~PageTableManager(); + + private: + /** + * Local copy of Page Table for user-space testing + * (set to NULL for kernel instance) + */ + char* ivTABLE; + + + /** + * Represents a single entry in the page table + */ + struct PageTableEntry + { + /** + * Dword0 + */ + union { + struct { /**< Dword0 Attributes */ + uint64_t B:2; /**< 0:1 Segment Size */ + uint64_t AVA:55; /**< 2:56 Abbreviated Virtual Address = VA w/o bottom 23 bits */ + uint64_t SW:2; /**< 57:58 =SW[0:1] - Reserved */ + uint64_t LRU:2; /**< 59:60 =SW[2:3] - Used for LRU algorithm */ + uint64_t L:1; /**< 61 Virtual page size */ + uint64_t H:1; /**< 62 Hash function identifier */ + uint64_t V:1; /**< 63 Entry valid */ + }; + uint64_t dword0; /**< Full Dword0 */ + }; + + /** + * Dword1 + */ + union { + struct { /**< Dword0 Attributes */ + uint64_t pp0:1; /**< 0 Page Protection bit 0 */ + uint64_t rsv:1; /**< 1 Reserved */ + uint64_t key0_1:2; /**< 2:3 KEY bits 0:1 */ + uint64_t PN:48; /**< 4:52 Abbreviated Real Page Number + Large page size selector */ + uint64_t key2_4:3; /**< 53:54 KEY bits 2:4 */ + uint64_t R:1; /**< 55 Reference bit */ + uint64_t C:1; /**< 56 Change bit */ + uint64_t WIMG:4; /**< 57:60 Storage control bits */ + uint64_t N:1; /**< 61 No-execute page (N==1) */ + uint64_t pp1_2:2; /**< 62:63 Page Protection bits 1:2 */ + }; + uint64_t dword1; /**< Full Dword1 */ + }; + } PACKED; + + /** + * Internal Constants + */ + enum { + PTE_ACCESS_BITS = 0x800000000000007B, /**< pp0 + WIMG + pp1_2 */ + PTEG_SIZE_BYTES = (sizeof(PageTableEntry)*8), /**< Size of PTE Group in bytes */ + + SLBE_b = 12, /**< Page Size in bits per SLBE */ + SLBE_s = 40, /**< Segment Size in bits per SLBE */ + + }; + + /** + * @brief Return the hash value of the VA + * + * @param[in] i_vAddr Virtual Address to hash + * + * @return uint64_t hash value used by Page Table hardware + */ + uint64_t computeHash( uint64_t i_vAddr ); + + /** + * @brief Find the real address of the PTEG that matches the given VA + * + * @param[in] i_vAddr Virtual Address to look up + * + * @return uint64_t PTEG address + */ + uint64_t findPTEG( uint64_t i_vAddr ); + + /** + * @brief Find the PTE that matches the given VA + * + * @param[in] i_vAddr Virtual Address to search for + * + * @return PageTableEntry* Pointer to PTE, this is a real address + */ + PageTableEntry* findPTE( uint64_t i_vAddr ); + + /** + * @brief Find the real address of the PTE that matches the given address + * + * @param[in] i_vAddr Virtual Address to search for + * @param[in] i_ptegAddr Real Address of PTEG that would own this PTE + * + * @return PageTableEntry* Pointer to PTE, this is a real address + */ + PageTableEntry* findPTE( uint64_t i_vAddr, + uint64_t i_ptegAddr ); + + /** + * @brief Find the real address of a PTE that that is empty or invalid + * + * @param[in] i_ptegAddr Real Address of PTEG that would own this PTE + * + * @return PageTableEntry* Pointer to PTE slot, this is a real address + */ + PageTableEntry* findEmptyPTE( uint64_t i_ptegAddr ); + + /** + * @brief Find the real address of a PTE that can be invalidated + * and replaced + * + * @param[in] i_ptegAddr Real Address of PTEG that would own this PTE + * + * @return PageTableEntry* Pointer to PTE, this is a real address + */ + PageTableEntry* findOldPTE( uint64_t i_ptegAddr ); + + /** + * @brief Write a PTE to memory and update caches appropriately + * + * @param[in] i_pte Local pointer to PTE data + * @param[in] i_dest Real Address inside page table to write i_pte data into + * @param[in] i_valid true=set Valid bit, false=clear Valid bit + */ + void writePTE( PageTableEntry* i_pte, + PageTableEntry* i_dest, + bool i_valid ); + + /** + * @brief Add an entry to the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be mapped + * @param[in] i_page Real physical page number to map to + * @param[in] i_accessType Type of access page will be given + */ + void _addEntry( uint64_t i_vAddr, + uint64_t i_page, + VmmManager::ACCESS_TYPES i_accessType ); + + /** + * @brief Remove an entry from the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be removed + */ + void _delEntry( uint64_t i_vAddr ); + + /** + * @brief Remove an entry from the hardware page table + * + * @param[in] i_pte Pointer to real PTE in the page table + */ + void delEntry( PageTableEntry* i_pte ); + + /** + * @brief Remove a range of entries from the hardware page table + * + * @param[in] i_vAddrStart Beginning of VA range to remove + * @param[in] i_vAddrFinish End of VA range to remove + */ + void _delRangeVA( uint64_t i_vAddrStart, + uint64_t i_vAddrFinish ); + + /** + * @brief Remove a range of entries from the hardware page table + * + * @param[in] i_pnStart First Physical Page to remove + * @param[in] i_pnFinish Last Physical Page to remove + */ + void _delRangePN( uint64_t i_pnStart, + uint64_t i_pnFinish ); + + /** + * @brief Return status information about an entry in the hardware page table + * + * @param[in] i_vAddr Virtual Address within the page to be queried + * @param[out] o_pn Real physical page number to map to, INVALID_PN if PTE is invalid + * + * @return uint64_t ORed combination of status flags + */ + uint64_t _getStatus( uint64_t i_vAddr, + uint64_t& o_pn ); + + /** + * @brief Translate a PTE into the status bits + * + * @param[in] i_pte Pointer to PTE, could be local memory or part of page table + * + * @return uint64_t ORed combination of status flags + */ + uint64_t getStatus( PageTableEntry* i_pte ); + + /** + * @brief Update the LRU statistics for other PTEs in the same PTEG as the target + * + * @param[in] i_newPTE Real address of PTE that is being added to the Page Table + */ + void updateLRU( const PageTableEntry* i_newPTE ); + + /** + * @brief Invalidate TLB for a PTE + * + * @param[in] i_newPTE Real address of PTE that is being modified + */ + void invalidateTLB( PageTableEntry* i_pte ); + + /** + * @brief Invalidate all PTEs in the table + */ + void invalidatePT( void ); + + /** + * @brief Print out the contents of a PTE to the printk buffer + * + * @param[in] i_label string to display along with the PTE data + * @param[in] i_pteAddr pointer to the PTE to display + * @param[in] i_verbose true=break out all attributes, false=abbreviated output + */ + void printPTE( const char* i_label, + const PageTableEntry* i_pte, + bool i_verbose = false ); + + /** + * @brief Print out the entire Page Table to the printk buffer + */ + void _printPT( void ); + + /** + * @brief Return the real address of the page table in memory + * @return uint64_t Page Table Address + */ + uint64_t getAddress( void ); + + /** + * @brief Return the size of the page table in memory in bytes + * @return uint64_t Size of Page Table in bytes + */ + uint64_t getSize( void ); + + /** + * @brief Set bits in PTE for the given ACCESS_TYPES + * + * @param[out] o_pte PTE to modify + * @param[in] i_accessType Access parameter to apply to PTE + */ + void setAccessBits( PageTableEntry* o_pte, + VmmManager::ACCESS_TYPES i_accessType ); + + /** + * @brief Convert the bits from a PTE into a ACCESS_TYPES + * + * @param[in] i_pte PTE to examine + * + * @return ACCESS_TYPES Access parameters of given PTE + */ + VmmManager::ACCESS_TYPES getAccessType( const PageTableEntry* i_pte ); + + /** + * @brief Fill in default values for the PTE + * + * @param[out] o_pte PTE to modify + */ + static void setupDefaultPTE( PageTableEntry* o_pte ); + + + + // Allow testcase to see inside + friend class ptmgrtest; +}; + +#endif diff --git a/src/include/kernel/vmmmgr.H b/src/include/kernel/vmmmgr.H index 4fc378578..c194b0456 100644 --- a/src/include/kernel/vmmmgr.H +++ b/src/include/kernel/vmmmgr.H @@ -7,11 +7,6 @@ class VmmManager { public: - struct pte_t - { - uint64_t a,b; - }; - enum VMM_CONSTS { EIGHT_MEG = 8 * 1024 * 1024, @@ -19,10 +14,8 @@ class VmmManager FULL_MEM_SIZE = 1 * EIGHT_MEG, PAGESIZE = 4096, - PTSIZE = (1 << 18), - PTEG_SIZE = 8, - PTEG_COUNT = (PTSIZE / sizeof(pte_t)) / PTEG_SIZE, - + // put the Page Table at the end of our memory space + PTSIZE = (1 << 18), HTABORG = (FULL_MEM_SIZE - PTSIZE), }; @@ -32,6 +25,7 @@ class VmmManager READ_O_ACCESS, NORMAL_ACCESS, CI_ACCESS, + RO_EXE_ACCESS, }; enum PID_ALLOCATIONS @@ -70,90 +64,6 @@ class VmmManager void* _mmioMap(void*, size_t); int _mmioUnmap(void*, size_t); - static pte_t* page_table; - - inline volatile pte_t& getPte(uint64_t pteg, uint64_t idx) - { - return page_table[pteg * PTEG_SIZE + idx]; - } - - inline void defaultPte(volatile pte_t& pte) - { - pte.a = 0x4000000000000000; // B = 01 (1TB). - pte.b = 0x0; - } - - inline void setValid(bool valid, volatile pte_t& pte) - { - // Adding a page requires EIEIO to ensure update of PTE prior - // to making valid and PTESYNC afterwards. - // Removing a page just requires PTESYNC afterwards. - if (valid) - { - asm volatile("eieio" ::: "memory"); - } - - pte.a &= ~0x01; - pte.a |= (valid ? 0x1 : 0x0); - - if (!valid) - { - asm volatile("ptesync" ::: "memory"); - - register uint64_t rS = 0, rB = 0; - rB = (getTid(pte) << 11) // VA[0:54). - | (((uint64_t)&pte) & 0x3FF8) >> 7; // VA[55:65]. - rB <<= 12; // Put in rB[0:51]. - rB |= 0x0100; // B = 01 (1TB). - - // TLBIE isn't correct in gcc, hand code asm. - asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 21)" :: - "r"(rB), "r"(rS) : "memory"); - - asm volatile("eieio" ::: "memory"); - asm volatile("tlbsync" ::: "memory"); - } - - asm volatile("ptesync" ::: "memory"); - } - - inline bool isValid(volatile pte_t& pte) - { - return ((pte.a & 0x01) == 0x01); - } - - inline void setTid(tid_t tid, volatile pte_t& pte) - { - pte.a &= 0xC00000000000007F; - pte.a |= ((uint64_t) tid) << 7; - } - - inline tid_t getTid(volatile pte_t& pte) - { - return (tid_t) ((pte.a & 0xC00000000000007F) >> 7); - } - - inline void setAccess(ACCESS_TYPES t, volatile pte_t& pte) - { - uint64_t pteMask = ~0x800000000000007B; - pte.b &= pteMask; - pte.b |= (NO_USER_ACCESS == t ? 0x0000000000000010 : - (READ_O_ACCESS == t ? 0x0000000000000011 : - (NORMAL_ACCESS == t ? 0x0000000000000012 : - (CI_ACCESS == t ? 0x000000000000002A : - 0x0)))); - } - - inline void setPage(uint64_t page, volatile pte_t& pte) - { - pte.b &= ~0x0FFFFFFFFFFFF000; - pte.b |= page << 12; - } - - inline uint64_t getPage(volatile pte_t& pte) - { - return (pte.b & 0x0FFFFFFFFFFFF000) >> 12; - } }; #endif diff --git a/src/kernel/makefile b/src/kernel/makefile index 4f100c660..fd64fd611 100644 --- a/src/kernel/makefile +++ b/src/kernel/makefile @@ -1,7 +1,7 @@ ROOTPATH = ../.. OBJS = start.o kernel.o console.o pagemgr.o heapmgr.o taskmgr.o cpumgr.o -OBJS += syscall.o scheduler.o spinlock.o exception.o vmmmgr.o timemgr.o futexmgr.o +OBJS += syscall.o scheduler.o spinlock.o exception.o vmmmgr.o timemgr.o futexmgr.o ptmgr.o include ${ROOTPATH}/config.mk diff --git a/src/kernel/ptmgr.C b/src/kernel/ptmgr.C new file mode 100644 index 000000000..b4cce308a --- /dev/null +++ b/src/kernel/ptmgr.C @@ -0,0 +1,936 @@ +#include +#include +#include +#include +#include +#include + +//#define Dprintk(...) printk(args...) +#define Dprintk(args...) +#define Tprintk(args...) +#define Eprintk(args...) printk(args) + +// Utilities to do some bit manipulation + +/** + * @brief Extract a set of bits and right-justify the result + * @param i_var64[in] 64-bit word to extract data from + * @param i_startbit[in] Bit to start extraction from + * @param i_lastbit[in] Bit to stop extraction on, inclusive + * @return uint64_t Right-justified data + */ +ALWAYS_INLINE uint64_t EXTRACT_RJ( uint64_t i_var64, + uint64_t i_startbit, + uint64_t i_lastbit ) +{ + uint64_t mask = ((0xFFFFFFFFFFFFFFFF >> i_startbit) & (0xFFFFFFFFFFFFFFFF << (63 - i_lastbit))); + uint64_t result = (i_var64 & mask) >> (63 - i_lastbit); + return result; +} + +/** + * @brief Extract a set of bits and left-justify the result + * @param i_var64[in] 64-bit word to extract data from + * @param i_startbit[in] Bit to start extraction from + * @param i_lastbit[in] Bit to stop extraction on, inclusive + * @return uint64_t Left-justified data + */ +ALWAYS_INLINE uint64_t EXTRACT_LJ( uint64_t var64, + uint64_t i_startbit, + uint64_t i_lastbit ) +{ + uint64_t mask = ((0xFFFFFFFFFFFFFFFF >> i_startbit) & (0xFFFFFFFFFFFFFFFF << (63 - i_lastbit))); + uint64_t result = (var64 & mask) << i_startbit; + return result; +} + +/** + * @brief Extract a set of bits from the last word of data and right-justify the result + * + * Example: Extract bits 25:54 from a 79 bit buffer : + * bb = EXTRACT_RJ_LEN(aa,79,25,54) + * + * @param i_lastword[in] Right-most 64-bit word of larger data buffer, data[len-64:len] + * @param i_bitlen[in] Total number of bits in original buffer + * @param i_startbit[in] Bit to start extraction from, relative to original bit length + * @param i_lastbit[in] Bit to stop extraction on, inclusive, relative to original bit length + * @return uint64_t Left-justified data + */ +ALWAYS_INLINE uint64_t EXTRACT_RJ_LEN( uint64_t i_lastword, + uint64_t i_bitlen, + uint64_t i_startbit, + uint64_t i_lastbit ) +{ + Dprintk( "i_lastword=%.16lX, i_bitlen=%ld, i_startbit=%ld, i_lastbit=%ld\n", i_lastword, i_bitlen, i_startbit, i_lastbit ); + if( (i_lastbit - i_startbit) > 64 ) + { + Eprintk("error %d : i_lastword=%.16lX, i_bitlen=%ld, i_startbit=%ld, i_lastbit=%ld\n", __LINE__, i_lastword, i_bitlen, i_startbit, i_lastbit); + kassert(false); + } + else if( i_lastbit >= i_bitlen ) + { + Eprintk("error %d : i_lastword=%.16lX, i_bitlen=%ld, i_startbit=%ld, i_lastbit=%ld\n", __LINE__, i_lastword, i_bitlen, i_startbit, i_lastbit); + kassert(false); + } + else if( i_bitlen <= 64 ) + { + uint64_t diff = 64 - i_bitlen; + return EXTRACT_RJ( i_lastword, i_startbit + diff, i_lastbit + diff ); + } + else if( i_lastbit < (i_bitlen - 64) ) + { + // desired bits are inside the first word + return 0; + } + + // goal is to left-justify the i_startbit to be bit0 in the resulting word + uint64_t diff = i_bitlen - 64; //=bits to the left of i_lastword + + if( i_startbit < diff ) + { + //move the buffer to the right, zeros will fill in the extra bits + i_lastword = i_lastword >> (diff - i_startbit); + } + else + { + //move the buffer to the left to justify it + i_lastword = i_lastword << (i_startbit - diff); + } + + i_lastbit -= i_startbit; + i_startbit = 0; + + + return EXTRACT_RJ( i_lastword, i_startbit, i_lastbit ); +} + +/** + * @brief Extract a set of bits from the last word of data and left-justify the result + * + * @param i_lastword[in] Right-most 64-bit word of larger data buffer, data[len-64:len] + * @param i_bitlen[in] Total number of bits in original buffer + * @param i_startbit[in] Bit to start extraction from, relative to original bit length + * @param i_lastbit[in] Bit to stop extraction on, inclusive, relative to original bit length + * @return uint64_t Left-justified data + */ +ALWAYS_INLINE uint64_t EXTRACT_LJ_LEN( uint64_t i_lastword, uint64_t i_bitlen, uint64_t i_startbit, uint64_t i_lastbit ) +{ + uint64_t diff = i_bitlen - 64; + i_lastword = i_lastword >> diff; + if( i_lastbit < 64 ) { + i_lastbit = i_lastbit - diff; + } else { + Eprintk("error %d : i_lastword=%lX, i_bitlen=%ld, i_startbit=%ld, i_lastbit=%ld\n", __LINE__, i_lastword, i_bitlen, i_startbit, i_lastbit); + kassert(false); + } + if( i_startbit < 64 ) { + i_startbit = i_startbit - diff; + } else { + Eprintk("error %d : i_lastword=%lX, i_bitlen=%ld, i_startbit=%ld, i_lastbit=%ld\n", __LINE__, i_lastword, i_bitlen, i_startbit, i_lastbit); + kassert(false); + } + return EXTRACT_LJ( i_lastword, i_startbit, i_lastbit ); +} + +/******************** + Public Methods + ********************/ + +/** + * STATIC + * @brief Static Initializer + */ +void PageTableManager::init() +{ + Singleton::instance(); +} + +/** + * STATIC + * @brief Add an entry to the hardware page table + */ +void PageTableManager::addEntry( uint64_t i_vAddr, + uint64_t i_page, + VmmManager::ACCESS_TYPES i_accessType ) +{ + return Singleton::instance()._addEntry( i_vAddr, i_page, i_accessType ); +} + +/** + * STATIC + * @brief Remove an entry from the hardware page table + */ +void PageTableManager::delEntry( uint64_t i_vAddr ) +{ + return Singleton::instance()._delEntry(i_vAddr); +} + +/** + * STATIC + * @brief Remove a range of entries from the hardware page table + */ +void PageTableManager::delRangeVA( uint64_t i_vAddrStart, + uint64_t i_vAddrFinish ) +{ + return Singleton::instance()._delRangeVA(i_vAddrStart,i_vAddrFinish); +} + +/** + * STATIC + * @brief Remove a range of entries from the hardware page table + */ +void PageTableManager::delRangePN( uint64_t i_pnStart, + uint64_t i_pnFinish ) +{ + return Singleton::instance()._delRangePN(i_pnStart,i_pnFinish); +} + + +/** + * STATIC + * @brief Return status information about an entry in the hardware page table + */ +uint64_t PageTableManager::getStatus( uint64_t i_vAddr, + uint64_t& o_pn ) +{ + return Singleton::instance()._getStatus(i_vAddr,o_pn); +} + +/** + * STATIC + * @brief Print out the contents of a PTE + */ +void PageTableManager::printPTE( const char* i_label, + uint64_t i_pteAddr, + bool i_verbose ) +{ + Singleton::instance().printPTE( i_label, (PageTableEntry*)i_pteAddr, i_verbose ); +} + +/** + * STATIC + * @brief Print out the contents of a PTE + */ +void PageTableManager::printPTE( uint64_t i_va, + bool i_verbose ) +{ + PageTableEntry* pte = Singleton::instance().findPTE(i_va); + Singleton::instance().printPTE( NULL, pte, i_verbose ); +} + +/** + * STATIC + * @brief Print out the entire Page Table + */ +void PageTableManager::printPT( void ) +{ + Singleton::instance()._printPT(); +} + + +/******************** + Private/Protected Methods + ********************/ + +/** + * @brief Constructor + */ +PageTableManager::PageTableManager( bool i_userSpace ) +: ivTABLE(NULL) +{ + if( i_userSpace ) + { + ivTABLE = new char[getSize()]; + //printk( "** PageTableManager running in USER_SPACE : ivTABLE = %p**\n", ivTABLE ); + } + else + { + printk( "Page Table is at 0x%.16lX : 0x%.16lX\n", getAddress(), getAddress() + getSize() ); + } + + //initialize the table to be invalid + invalidatePT(); +} + +/** + * @brief Invalidate all PTEs in the table + */ +void PageTableManager::invalidatePT( void ) +{ + PageTableEntry* pte = (PageTableEntry*)getAddress(); + uint64_t num_ptes = getSize() / sizeof(PageTableEntry); + for( uint64_t x = 0; x < num_ptes; x++ ) + { + pte->AVA = 0xFFFFFFFFFFFF; + pte->V = 0; + pte++; + } +} + +PageTableManager::~PageTableManager() +{ + if( ivTABLE ) { + delete[] ivTABLE; + } +} + + +/** + * @brief Add an entry to the hardware page table + */ +void PageTableManager::_addEntry( uint64_t i_vAddr, + uint64_t i_page, + VmmManager::ACCESS_TYPES i_accessType ) +{ + Tprintk( ">> PageTableManager::_addEntry( i_vAddr=0x%.16lX, i_page=%ld, i_accessType=%d )\n", i_vAddr, i_page, i_accessType ); + + //Note: no need to lock here because that is handled by higher function + + PageTableEntry pte_data; + setupDefaultPTE( &pte_data ); + + // find the matching PTEG first so we only do it once + uint64_t pteg_addr = findPTEG( i_vAddr ); + + //look for a matching entry in the table already + PageTableEntry* pte_slot = findPTE( i_vAddr, pteg_addr ); + if( pte_slot == NULL ) + { + // look for an empty/invalid entry that we can use + pte_slot = findEmptyPTE( pteg_addr ); + if( pte_slot == NULL ) + { + // look for a valid entry we can steal + pte_slot = findOldPTE( pteg_addr ); + } + } + else + { + if( (pte_slot->V == 1) && (i_page != pte_slot->PN) ) + { + Eprintk( "**ERROR** PageTableManager::_addEntry> Duplicate PTE with different page number\n" ); + kassert(false); + } + } + + // we can't handle any other cases... + if( pte_slot == NULL ) + { + Eprintk( "**ERROR** PageTableManager::_addEntry> Nowhere to put the new PTE\n" ); + kassert(false); + } + + // update the access bits in our local copy + setAccessBits( &pte_data, i_accessType ); + + // update the Abbreviated Virtual Address + pte_data.AVA = (i_vAddr >> 23); + + // update the Abbreviated Real Page Number + pte_data.PN = i_page; + + //Note: We are ignoring the LP field + + // write the new entry into mainstore + writePTE( &pte_data, pte_slot, true ); + + + Dprintk( "<< PageTableManager::_addEntry()\n" ); +} + +/** + * @brief Remove an entry from the hardware page table + */ +void PageTableManager::_delEntry( uint64_t i_vAddr ) +{ + // find the corresponding PTE + PageTableEntry* pte = findPTE( i_vAddr ); + if( pte ) + { + delEntry( pte ); + } +} + +/** + * @brief Remove an entry from the hardware page table + */ +void PageTableManager::delEntry( PageTableEntry* i_pte ) +{ + writePTE( i_pte, i_pte, false ); +} + +/** + * @brief Remove a range of entries from the hardware page table + */ +void PageTableManager::_delRangeVA( uint64_t i_vAddrStart, + uint64_t i_vAddrFinish ) +{ + // Note : this could potentially be very slow for large ranges + + // loop around 4K pages within the range + for( uint64_t va = i_vAddrStart; va < i_vAddrFinish; va += VmmManager::PAGESIZE ) + { + _delEntry( va ); + } +} + +/** + * @brief Remove a range of entries from the hardware page table + */ +void PageTableManager::_delRangePN( uint64_t i_pnStart, + uint64_t i_pnFinish ) +{ + // Since the range is likely going to be quite large, we are going to + // loop around every PTE rather than looping through all of the pages + // within the given range + uint64_t pt_addr = getAddress(); + PageTableEntry* pte = (PageTableEntry*) pt_addr; + uint64_t num_ptes = getSize() / sizeof(PageTableEntry); + for( uint64_t x = 0; x < num_ptes; x++ ) + { + if( (pte->V == 1) && (pte->PN >= i_pnStart) && (pte->PN <= i_pnFinish) ) + { + writePTE( pte, pte, false ); + } + + pte++; + } +} + +/** + * @brief Return status information about an entry in the hardware page table + */ +uint64_t PageTableManager::_getStatus( uint64_t i_vAddr, + uint64_t& o_pn ) +{ + PageTableEntry* pte = findPTE( i_vAddr ); + o_pn = INVALID_PN; + if( pte ) { + o_pn = pte->PN; + } + return getStatus( pte ); +} + +/** + * @brief Translate a PTE into the status bits + */ +uint64_t PageTableManager::getStatus( PageTableEntry* i_pte ) +{ + if( i_pte == NULL ) + { + return PTE_UNKNOWN; + } + + // translate different bits in the struct + uint64_t status = PTE_UNKNOWN; + status |= PTE_PRESENT; + if( i_pte->V == 1 ) { + status |= PTE_VALID; + } + + VmmManager::ACCESS_TYPES access = getAccessType(i_pte); + switch( access ) { + case( VmmManager::CI_ACCESS ): + status |= PTE_CACHE_INHIBITED; + break; + case( VmmManager::READ_O_ACCESS ): + status |= PTE_READ_ONLY; + break; + case( VmmManager::NORMAL_ACCESS ): + status |= PTE_EXECUTE; //@fixme? + break; + case( VmmManager::RO_EXE_ACCESS ): + status |= PTE_READ_ONLY; + status |= PTE_EXECUTE; + break; + default: + break; + }; + + if( i_pte->C == 1 ) { + status |= PTE_MODIFIED; + } + if( i_pte->R == 1 ) { + status |= PTE_ACCESSED; + } + + return status; +} + + +/** + * @brief Return the 39-bit hash value of the VA + */ +uint64_t PageTableManager::computeHash( uint64_t i_vAddr ) +{ + //Note: VA is really 78-bits, we are assuming top 14 bits are always zero + + uint64_t l_hash_val = 0; + + if( SLBE_s == 40 ) + { + // the hash value is computed by + // Exclusive ORing the following three quantities: + // (VA[24:37]||<25>0), (0||VA[0:37]), and (0||VA[38:77-b]) + + //mask off unrelated bits, right-justify bit 37, append 25 zeros + uint64_t va__24_37 = EXTRACT_RJ_LEN( i_vAddr, 78, 24, 37 ) << 25; + //mask off unrelated bits, right-justify bit 37 + uint64_t va__0_37 = EXTRACT_RJ_LEN( i_vAddr, 78, 0, 37 ); + //mask off unrelated bits, right-justify bit 65 (77-12) + uint64_t va__38_77b = EXTRACT_RJ_LEN( i_vAddr, 78, 38, 77-SLBE_b ); + + l_hash_val = va__24_37 ^ va__0_37 ^ va__38_77b; + + Dprintk( "computeHash(i_vAddr=0x%.16lX)\n", i_vAddr ); + Dprintk( "va__24_37 = 0x%.16lX\n", va__24_37 ); + Dprintk( "va__0_37 = 0x%.16lX\n", va__0_37 ); + Dprintk( "va__38_77b = 0x%.16lX\n", va__38_77b ); + + } + else if( SLBE_s == 28 ) + { + // the hash value is computed by + // Exclusive ORing VA[11:49] with (<11+b>0||VA[50:77-b]) + + //mask off unrelated bits, right-justify bit 49 + uint64_t va__11_49 = EXTRACT_RJ_LEN( i_vAddr, 78, 11, 49 ); + //mask off unrelated bits, right-justify bit 65 (77-12) + uint64_t va__50_77b = EXTRACT_RJ_LEN( i_vAddr, 78, 50, 77-SLBE_b ); + + l_hash_val = va__11_49 ^ va__50_77b; + } + + //Note: not using Secondary Hash (LPCR[TC]==1) + + + Dprintk( "l_hash_val = 0x%.16lX\n", l_hash_val ); + return l_hash_val; +} + +/** + * @brief Find the 60-bit real address of the PTEG that matches the given virtual address + */ +uint64_t PageTableManager::findPTEG( uint64_t i_vAddr ) +{ + // hash is an index into a virtual array of PTEGs + uint64_t hash = computeHash(i_vAddr); //right-justified + + // mask off the hash to fit into the table + hash = hash % PTEG_COUNT; + + // use the hash as the index into the array of PTEGs + uint64_t pteg_addr = getAddress() + hash * PTEG_SIZE_BYTES; + + Dprintk( "PageTableManager::findPTEG(i_vAddr=0x%.16lX) = 0x%.16lX\n", i_vAddr, pteg_addr ); + return pteg_addr; +} + +/** + * @brief Find the real address of the PTE that matches the given address + */ +PageTableManager::PageTableEntry* PageTableManager::findPTE( uint64_t i_vAddr ) +{ + Dprintk( ">> PageTableManager::findPTE(i_vAddr=0x%.16lX)\n", i_vAddr ); + + // first find the PTEG + uint64_t pteg_addr = findPTEG( i_vAddr ); + + // look for a PTE in that PTEG + PageTableEntry* pte_found = findPTE( i_vAddr, pteg_addr ); + + Dprintk( "PageTableManager::findPTE() = %.16lX <<\n", (uint64_t)pte_found ); + return pte_found; +} + +/** + * @brief Find the real address of the PTE that matches the given address + */ +PageTableManager::PageTableEntry* PageTableManager::findPTE( uint64_t i_vAddr, + uint64_t i_ptegAddr ) +{ + Tprintk( "PageTableManager::findPTE(i_vAddr=0x%.16lX,i_ptegAddr=0x%.16lX)>>\n", i_vAddr, i_ptegAddr ); + + PageTableEntry* pte_found = NULL; + + PageTableEntry* pte_cur = (PageTableEntry*)i_ptegAddr; + + // loop through all 8 PTEs + for( uint64_t x = 0; x < 8; x++ ) + { + // compare input to AVA + //2:56 Abbreviated Virtual Address = VA w/o bottom 23 bits + if( pte_cur->AVA == (i_vAddr >> 23) ) + { + Tprintk( "Found match at PTE #%ld\n", x ); + //printPTE( pte_cur ); + pte_found = pte_cur; + break; + } + + pte_cur++; + } + + Dprintk( "<>\n", (uint64_t)pte_found ); + return pte_found; +} + +/** + * @brief Write a PTE to memory and update caches appropriately + */ +void PageTableManager::writePTE( PageTableEntry* i_pte, + PageTableEntry* i_dest, + bool i_valid ) +{ + // are we stealing a PTE + bool pte_stolen = false; + if( (i_dest->V == 1) && i_valid ) + { + pte_stolen = true; + printPTE( "Stealing", i_dest ); + } + + if(ivTABLE) + { + Dprintk( ">> PageTableManager::writePTE( i_dest=%p, i_valid=%d ) **FAKE**\n", i_dest, i_valid ); + memcpy( (void*) i_dest, (void*) i_pte, sizeof(PageTableEntry) ); + if( i_valid ) { + i_dest->V = 1; + //printPTE( "Writing", i_dest ); + } else { + i_dest->V = 0; + //printk( ">> PageTableManager::writePTE( i_dest=%p, i_valid=%d ) **FAKE**\n", i_dest, i_valid ); + //printPTE( "Removing", i_dest ); + } + } + else + { + Dprintk( ">> PageTableManager::writePTE( i_dest=0x%.lX, i_valid=%d )\n", i_dest, i_valid ); + + if( i_valid ) { + // printPTE( "Writing", i_dest ); + } else { + printPTE( "Removing", i_dest ); + } + + i_dest->V = 0; /* (other fields don't matter) */ + + /* order update before tlbie and before next Page Table search */ + asm volatile("ptesync" ::: "memory"); + + // tlbie, eieio, tlbsync, ptesync + invalidateTLB(i_pte); + + // if we're removing an entry we can ignore the other fields + if( i_valid ) + { + //PTE:ARPN,LP,AC,R,C,WIMG,N,PP set to new values + i_dest->dword1 = i_pte->dword1; + + asm volatile("eieio" ::: "memory"); /* order 2nd update before 3rd */ + + //PTE:B,AVA,SW,L,H,V set to new values (V=1) + i_dest->dword0 = i_pte->dword0; + i_dest->LRU = 0; + i_dest->V = 1; + + /* order 2nd and 3rd updates before next Page Table search + and before next data access */ + asm volatile("ptesync" ::: "memory"); + } + } + + // update the other entries' LRU statistics + updateLRU( i_dest ); + + Dprintk( "<< PageTableManager::writePTE()\n" ); +} + + +/** + * @brief Print out the contents of a PTE + */ +void PageTableManager::printPTE( const char* i_label, + const PageTableEntry* i_pte, + bool i_verbose ) +{ + if( i_pte == NULL ) + { + if( i_label ) { printk( "%s :: ", i_label ); } + printk( "NULL PTE\n" ); + return; + } + + uint64_t pte_num = (((uint64_t)i_pte) - getAddress()) / sizeof(PageTableEntry); + + if( i_label ) { printk( "%s :: ", i_label ); } + if( i_verbose ) + { + printk( "[%4ld:%4ld]> @%p\n", pte_num/PTEG_SIZE, pte_num%PTEG_SIZE, i_pte ); + printk( "Dword : %.16lX %.16lX\n", ((uint64_t*)i_pte)[0], ((uint64_t*)i_pte)[1] ); + printk( "-AVA : 0x%.14lX\n", i_pte->AVA ); + printk( "-SW : %ld\n", i_pte->SW ); + printk( "-LRU : %ld\n", i_pte->LRU ); + printk( "-V : %ld\n", i_pte->V ); + printk( "-RC : %ld%ld\n", i_pte->R, i_pte->C ); + printk( "-WIMG : 0x%.1lX\n", i_pte->WIMG ); + printk( "-pp0 : %ld\n", i_pte->pp0 ); + printk( "-pp1_2 : %ld\n", i_pte->pp1_2 ); + printk( "-PN : %ld\n", i_pte->PN ); + } + else + { + printk( "[%4ld:%4ld]> @%p : %.16lX %.16lX : AVA=%16lX, PN=%ld\n", pte_num/PTEG_SIZE, pte_num%PTEG_SIZE, i_pte, i_pte->dword0, i_pte->dword1, i_pte->AVA, i_pte->PN ); + } + +} + + +/** + * @brief Print out the entire Page Table + */ +void PageTableManager::_printPT( void ) +{ + printk( "- -Page Table --\n" ); + uint64_t pt_addr = getAddress(); + PageTableEntry* pte = (PageTableEntry*) pt_addr; + printk( "@%p..0x%.16lX\n", pte, pt_addr + getSize() ); + + uint64_t num_ptes = getSize() / sizeof(PageTableEntry); + for( uint64_t x = 0; x < num_ptes; x++ ) + { + if( pte->V == 1 ) + { + printPTE( NULL, pte ); + } + + pte++; + } + + printk( "-- End Page Table --\n" ); +} + +/** + * @brief Return the real address of the page table in memory + */ +uint64_t PageTableManager::getAddress( void ) +{ + if(ivTABLE) { + return (uint64_t)ivTABLE; + } else { + return VmmManager::HTABORG; + } +} + +/** + * @brief Return the size of the page table in memory in bytes + */ +inline uint64_t PageTableManager::getSize( void ) +{ + return (256*1024); //256KB +} + +/** + * @brief Set bits in PTE for the given ACCESS_TYPES + */ +void PageTableManager::setAccessBits( PageTableEntry* o_pte, + VmmManager::ACCESS_TYPES i_accessType ) +{ + o_pte->dword1 &= ~PTE_ACCESS_BITS; + if( VmmManager::NO_USER_ACCESS == i_accessType ) { + o_pte->WIMG = 0b0010; // Memory Coherency Required + } else if( VmmManager::READ_O_ACCESS == i_accessType ) { + o_pte->WIMG = 0b0010; // Memory Coherency Required + o_pte->pp1_2 = 0b01; // PP=001 + } else if( VmmManager::NORMAL_ACCESS == i_accessType ) { + o_pte->WIMG = 0b0010; // Memory Coherency Required + o_pte->pp1_2 = 0b10; // PP=010 + } else if( VmmManager::CI_ACCESS == i_accessType ) { + o_pte->WIMG = 0b0101; // Cache Inhibited, Guarded + o_pte->pp1_2 = 0b10; // PP=010 + } else { + //@fixme - add RO_EXE_ACCESS + Eprintk( "** unrecognized access=%d\n", i_accessType ); + } +} + +/** + * @brief Convert the bits from a PTE into a ACCESS_TYPES + */ +VmmManager::ACCESS_TYPES PageTableManager::getAccessType( const PageTableEntry* i_pte ) +{ + if( i_pte->pp0 == 0b0 ) + { + if( (i_pte->WIMG == 0b0101) && (i_pte->pp1_2 == 0b10) ) + { + return VmmManager::CI_ACCESS; + } + else if( (i_pte->WIMG == 0b0010) && (i_pte->pp1_2 == 0b00) ) + { + return VmmManager::NO_USER_ACCESS; + } + else if( (i_pte->WIMG == 0b0010) && (i_pte->pp1_2 == 0b01) ) + { + return VmmManager::READ_O_ACCESS; + } + else if( (i_pte->WIMG == 0b0010) && (i_pte->pp1_2 == 0b10) ) + { + return VmmManager::NORMAL_ACCESS; + } + //@fixme - add RO_EXE_ACCESS + } + + Eprintk( "I don't recognize this PTE : WIMG=%ld, pp1_2=%ld\n", i_pte->WIMG, i_pte->pp1_2 ); + printPTE( "getAccessType", i_pte); + kassert(false); + return VmmManager::NO_USER_ACCESS; +} + +/** + * @brief Fill in default values for the PTE + */ +void PageTableManager::setupDefaultPTE( PageTableEntry* o_pte ) +{ + o_pte->B = 0b01; //Segment Size (01=1TB) + o_pte->L = 0b0; //Virtual page size (1=>4KB) + o_pte->H = 0b0; //Hash function identifier (0=primary hash) +} + +/** + * @brief Find the real address of a PTE that that is empty or invalid + */ +PageTableManager::PageTableEntry* PageTableManager::findEmptyPTE( uint64_t i_ptegAddr ) +{ + Tprintk( "PageTableManager::findEmptyPTE(i_ptegAddr=0x%.16lX)>>\n", i_ptegAddr ); + + PageTableEntry* pte_slot = NULL; + PageTableEntry* pte_cur = (PageTableEntry*)i_ptegAddr; + + // loop through all 8 PTEs + for( uint64_t x = 0; x < 8; x++ ) + { + // look for an invalid entry + if( pte_cur->V == 0 ) + { + Tprintk( "Found invalid slot at #%ld\n", x ); + //printPTE( pte_cur ); + pte_slot = pte_cur; + break; + } + + pte_cur++; + } + + Dprintk( "<>\n", pte_slot ); + return pte_slot; +} + +/** + * @brief Find the real address of a PTE that can be invalidated + * and replaced + */ +PageTableManager::PageTableEntry* PageTableManager::findOldPTE( uint64_t i_ptegAddr ) +{ + // Order of preference for PTE slots to steal: + // 1) PTE with highest use count (LRU==SW[2:3]) + // 2) Lowest PTE with the highest use count + PageTableEntry* pte = (PageTableEntry*)i_ptegAddr; + PageTableEntry* old_pte = pte; + for( uint64_t x = 0; x < 8; x++ ) + { + if( pte->LRU > old_pte->LRU ) + { + old_pte = pte; + } + + pte++; + } + PageTableManager::printPTE( "Dropping PTE", old_pte ); + + return old_pte; +} + +/** + * @brief Update the LRU statistics for other PTEs in the same PTEG as the target + */ +void PageTableManager::updateLRU( const PageTableEntry* i_newPTE ) +{ + Tprintk( ">> PageTableManager::updateLRU( i_newPTE=%p )\n", i_newPTE ); + + // find the beginning of the PTEG + uint64_t pteg_addr = (((uint64_t)i_newPTE) - getAddress()) / PTEG_SIZE_BYTES; + pteg_addr = pteg_addr*PTEG_SIZE_BYTES + getAddress(); + + // loop through all 8 PTEs in the PTEG + PageTableEntry* pte_cur = (PageTableEntry*)pteg_addr; + for( uint64_t x = 0; x < 8; x++ ) + { + // skip the entry we just added + if( pte_cur != i_newPTE ) + { + PageTableEntry new_pte = *pte_cur; + PageTableEntry old_pte = *pte_cur; + + // use this funny loop to avoid races where another thread + // is causing the C bit to be updated at the same time + do { + new_pte = *pte_cur; + old_pte = *pte_cur; + + if( (new_pte.V == 1) && (new_pte.R == 1) ) + { + new_pte.LRU = 1; + new_pte.R = 0; + } + else + { + if( new_pte.LRU < 0b11 ) + { + new_pte.LRU++; + } + } + } while( !__sync_bool_compare_and_swap( &(pte_cur->dword0), + old_pte.dword0, + new_pte.dword0 ) ); + + // tlbie, eieio, tlbsync, ptesync + invalidateTLB(pte_cur); + } + + pte_cur++; + } + + Tprintk( "<< PageTableManager::updateLRU(\n" ); +} + +/** + * @brief Invalidate TLB for a PTE + */ +void PageTableManager::invalidateTLB( PageTableEntry* i_pte ) +{ + Tprintk( ">> PageTableManager::invalidateTLB( i_pte=%p )\n", i_pte ); + + if( ivTABLE == NULL ) + { + /*invalidate old translation*/ + //tlbie(old_B,old_VA[14:77-b],old_L,old_LP,old_AP,old_LPID); + // TLBIE isn't correct in gcc, hand code asm. + register uint64_t rS = 0, rB = 0; + rB = (i_pte->AVA * 0x000FFFFFFFFFFFFF) << 12; // Put in rB[0:51] + rB |= 0x0100; // B = 01 (1TB). + asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 21)" :: + "r"(rB), "r"(rS) : "memory"); + + /* order tlbie before tlbsync */ + asm volatile("eieio" ::: "memory"); + + /* order tlbie before ptesync */ + asm volatile("tlbsync" ::: "memory"); + + /* order tlbie, tlbsync and 1st update before 2nd update */ + asm volatile("ptesync" ::: "memory"); + } + + Tprintk( "<< PageTableManager::invalidateTLB( )\n" ); +} + diff --git a/src/kernel/vmmmgr.C b/src/kernel/vmmmgr.C index 0fce50b7e..ba1191d90 100644 --- a/src/kernel/vmmmgr.C +++ b/src/kernel/vmmmgr.C @@ -2,6 +2,7 @@ #include #include #include +#include extern void* data_load_address; @@ -11,7 +12,7 @@ VmmManager::VmmManager() : lock() void VmmManager::init() { - printk("Starting VMM..."); + printk("Starting VMM...\n"); VmmManager& v = Singleton::instance(); @@ -19,7 +20,7 @@ void VmmManager::init() v.initPTEs(); v.initSDR1(); - printk("done.\n"); + printk("...done.\n"); }; void VmmManager::init_slb() @@ -69,12 +70,10 @@ void VmmManager::initSLB() void VmmManager::initPTEs() { - // Invalidate all. - for(size_t i = 0; i < PTEG_COUNT; i++) - for (size_t j = 0; j < PTEG_SIZE; j++) - setValid(false, getPte(i,j)); - - // Set up linear map. + // Initialize and invalidate the page table + PageTableManager::init(); + + // Set up linear map for every 4K page for(size_t i = 0; i < (FULL_MEM_SIZE / PAGESIZE); i++) { ACCESS_TYPES access = NORMAL_ACCESS; @@ -86,12 +85,8 @@ void VmmManager::initPTEs() { access = READ_O_ACCESS; } - volatile pte_t& pte = getPte(i % PTEG_COUNT, i / PTEG_COUNT); - defaultPte(pte); - setTid(i / PTEG_COUNT, pte); - setAccess(access, pte); - setPage(i, pte); - setValid(true, pte); + + PageTableManager::addEntry( i*PAGESIZE, i, access ); } } @@ -102,9 +97,6 @@ void VmmManager::initSDR1() asm volatile("mtsdr1 %0" :: "r"(sdr1) : "memory"); } -VmmManager::pte_t* VmmManager::page_table - = (VmmManager::pte_t*) HTABORG; - bool VmmManager::_pteMiss(task_t* t) { lock.lock(); @@ -137,22 +129,7 @@ bool VmmManager::_pteMiss(task_t* t) uint64_t mmioMapPage = mmioMapEntry / PAGESIZE; // Update PTE. - volatile pte_t& pte = getPte(effAddrPage, 1); - if ((getTid(pte) == effPid) && - (getPage(pte) == mmioMapPage) && - (isValid(pte))) - { - // Already present, maybe another thread. - lock.unlock(); - return true; - } - if (isValid(pte)) // Invalidate if already valid. - setValid(false, pte); - defaultPte(pte); - setTid(effPid, pte); - setPage(mmioMapPage, pte); - setAccess(CI_ACCESS, pte); - setValid(true, pte); + PageTableManager::addEntry( effAddr, mmioMapPage, CI_ACCESS ); lock.unlock(); return true; diff --git a/src/makefile b/src/makefile index 682def8f7..ddf8f743d 100644 --- a/src/makefile +++ b/src/makefile @@ -11,7 +11,8 @@ DIRECT_BOOT_OBJECTS = start.o kernel.o taskmgr.o cpumgr.o syscall.o \ scheduler.o exception.o vmmmgr.o timemgr.o \ syscall_stub.o syscall_task.o \ syscall_msg.o syscall_mmio.o syscall_time.o \ - init_main.o vfs_main.o sync.o futexmgr.o + init_main.o vfs_main.o sync.o futexmgr.o \ + ptmgr.o RUNTIME_OBJECTS = diff --git a/src/usr/testcore/kernel/ptmgrtest.H b/src/usr/testcore/kernel/ptmgrtest.H new file mode 100644 index 000000000..6ae1e2b4d --- /dev/null +++ b/src/usr/testcore/kernel/ptmgrtest.H @@ -0,0 +1,392 @@ +#ifndef __PTMGRTEST_H +#define __PTMGRTEST_H +/** + * @file ptmgr.H + * + * @brief Test cases for the Page Table Manager +*/ +#include +#include +#include +#include + +//#define PASS_TRACE(args...) TS_TRACE(args) +#define PASS_TRACE(args...) + +typedef struct pte_test_t { + uint64_t va; + uint64_t hash; + uint64_t page; + bool loaded; + bool delrangeva; +} pte_test_t; + + +const pte_test_t TEST_DATA[] = { + { 0x0000000000000100, 0x0000000000000000, 100, true, false }, // Page 0 + { 0x000000000053C008, 0x000000000000053C, 101, true, false }, // Something on the heap + { 0x0000000035004000, 0x0000000000035004, 102, true, false }, // <1TB + { 0x0000000040000000, 0x0000000000040000, 103, true, false }, // 1TB + { 0x0000000066666660, 0x0000000000066666, 104, true, false }, // 1TB < x < 2TB + { 0x0000000080000000, 0x0000000000080000, 105, true, false }, // 2TB + { 0x0000000080002000, 0x0000000000080002, 106, true, true }, // 2TB + 2 4K pages + { 0x0000000088888880, 0x0000000000088888, 107, true, false }, // 2TB < x < 3TB + { 0x00000000C0005000, 0x00000000000C0005, 108, true, false }, // >3TB + { 0x0000000040001FF8, 0x0000000000040001, 109, true, false }, // just before a page boundary + + //2 addresses in the same page + { 0x0000000090000040, 0x0000000000090000, 110, true, false }, + { 0x0000000090000100, 0x0000000000090000, 110, true, false }, + + //Out of range address (too big) ?? + { 0x0040000000000000, 0x0000000000004000, 111, true, false }, + + //Several addresses with the same PTEG, enough to overflow a PTEG + { 0x0000000080803000, 0x0000000000080803, 112, false, true }, // 2TB+8MB+12K + { 0x0000000081003000, 0x0000000000081003, 113, true, true }, // 2TB+16MB+12K + { 0x0000000082003000, 0x0000000000082003, 114, true, true }, // 2TB+32MB+12K + { 0x0000000084003000, 0x0000000000084003, 115, true, true }, // 2TB+64MB+12K + { 0x0000000084803000, 0x0000000000084803, 116, true, true }, // 2TB+72MB+12K + { 0x0000000085003000, 0x0000000000085003, 117, true, true }, // 2TB+80MB+12K + { 0x0000000085803000, 0x0000000000085803, 118, true, false }, // 2TB+88MB+12K + { 0x0000000086003000, 0x0000000000086003, 119, true, false }, // 2TB+96MB+12K + { 0x0000000086803000, 0x0000000000086803, 120, true, false }, // 2TB+124MB+12K +}; + + +class ptmgrtest : public CxxTest::TestSuite +{ + private: + enum { + VA_RANGE_START = 0x0000000080001000, + VA_RANGE_FINISH = 0x0000000085800000, + + PN_RANGE_START = 107, + PN_RANGE_FINISH = 115, + }; + + public: + + /** + * 1) Generate hash values for a range of addresses + * 2) Verify hash against hardcoded expected results + */ + void test_hash40( void ) + { + TS_TRACE( ">> test_hash40 <<" ); + printk( ">> test_hash40 <<\n" ); + uint64_t fails, total = 0; + + // Initialize the Page Table + PageTableManager* ptmgr = new PageTableManager(true); + + // test the hashes + uint64_t hash = 0; + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + hash = ptmgr->computeHash( TEST_DATA[x].va ); + if( TEST_DATA[x].hash != hash ) + { + TS_FAIL( "ptmgrtest::test_hash40> ERROR : Hash mismatch" ); + printk( "VA=0x%.16lX, Exp: %.16lX, Act: %.16lX\n", TEST_DATA[x].va, TEST_DATA[x].hash, hash ); + fails++; + } + else + { + PASS_TRACE( "ptmgrtest::test_hash40> PASS : 0x%.16lX", TEST_DATA[x].va ); + } + total++; + } + + delete ptmgr; + + TS_TRACE( "ptmgrtest::test_hash40> fails=%d/%d", fails, total ); + printk( "ptmgrtest::test_hash40> fails=%ld/%ld\n", fails, total ); + } + + /** + * 1) Initialize a local Page Table + * 2) Add some PTEs + * 3) Verify the PTE we just added is in the Page Table + * 4) Verify expected PTEs are still in the Page Table + */ + void test_addEntry( void ) + { + TS_TRACE( ">> test_addEntry <<" ); + printk( ">> test_addEntry <<\n" ); + uint64_t fails = 0; + uint64_t total = 0; + uint64_t status = 0; + uint64_t pn = 0; + + // 1) Initialize the Page Table + PageTableManager* ptmgr = new PageTableManager(true); + + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + // 2) Add some PTEs + ptmgr->_addEntry( TEST_DATA[x].va, TEST_DATA[x].page, VmmManager::NORMAL_ACCESS ); + + // 3) Verify the PTE we just added is in the Page Table + status = ptmgr->_getStatus( TEST_DATA[x].va, pn ); + if( !(status & PageTableManager::PTE_PRESENT) ) + { + TS_FAIL( "ptmgrtest::test_addEntry> ERROR1 : entry not found" ); + TS_TRACE( "Addr=%.16lX, Status=%.16lX", TEST_DATA[x].va, status ); + fails++; + } + else if( (status & PageTableManager::PTE_VALID) && (pn == TEST_DATA[x].page) ) + { + PASS_TRACE( "ptmgrtest::test_addEntry> PASS1 : 0x%.16lX", TEST_DATA[x].va ); + } + else + { + TS_FAIL( "ptmgrtest::test_addEntry> ERROR2 : unknown error" ); + TS_TRACE( "Addr=%.16lX, Status=%.16lX", TEST_DATA[x].va, status ); + fails++; + } + total++; + } + + // 4) Verify expected PTEs are still in the Page Table + //PRINT_PT; + + status = PageTableManager::PTE_UNKNOWN; + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + status = ptmgr->_getStatus( TEST_DATA[x].va, pn ); + + if( !(status & PageTableManager::PTE_PRESENT) && TEST_DATA[x].loaded ) + { + TS_FAIL( "ptmgrtest::test_addEntry> ERROR3 : entry not found" ); + TS_TRACE( "Addr=%.16lX, Status=%.16lX, Exp Page=%ld", TEST_DATA[x].va, status, TEST_DATA[x].page ); + fails++; + } + else if( (status & PageTableManager::PTE_VALID) + && !TEST_DATA[x].loaded ) + { + TS_FAIL( "ptmgrtest::test_addEntry> ERROR4 : PTE should be unloaded" ); + TS_TRACE( "Addr=%.16lX, Status=%.16lX", TEST_DATA[x].va, status ); + TS_TRACE( "Exp Page = %ld, Act Page = %ld", TEST_DATA[x].page, pn ); + fails++; + } + else if( (status & PageTableManager::PTE_VALID) + && (pn == TEST_DATA[x].page) + && TEST_DATA[x].loaded ) + { + PASS_TRACE( "ptmgrtest::test_addEntry> PASS2 : 0x%.16lX", TEST_DATA[x].va ); + } + else if( !(status & PageTableManager::PTE_VALID) + && !TEST_DATA[x].loaded ) + { + PASS_TRACE( "ptmgrtest::test_addEntry> PASS3 : 0x%.16lX", TEST_DATA[x].va ); + } + else + { + TS_FAIL( "ptmgrtest::test_addEntry> ERROR5 : unknown error" ); + TS_TRACE( "Addr=%.16lX, Status=%.16lX", TEST_DATA[x].va, status ); + TS_TRACE( "Exp Page = %ld, Act Page = %ld", TEST_DATA[x].page, pn ); + fails++; + } + total++; + } + + // delete our local table + delete ptmgr; + + TS_TRACE( "ptmgrtest::test_addEntry> fails=%d/%d", fails, total ); + printk( "ptmgrtest::test_addEntry> fails=%ld/%ld\n", fails, total ); + } + + /** + * 1) Initialize the Page Table + * 2) Populate the Page Table + * 3) Remove PTEs one at a time and verify they have been removed + */ + void test_delEntry( void ) + { + TS_TRACE( ">> test_delEntry <<" ); + printk( ">> test_delEntry <<\n" ); + uint64_t fails = 0; + uint64_t total = 0; + uint64_t ignored = 0; + + // 1) Initialize the Page Table + PageTableManager* ptmgr = new PageTableManager(true); + + // 2) Populate the Page Table + fillTable(ptmgr); + //ptmgr->_printPT(); + + // 3) Remove PTEs one at a time and verify they have been removed + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + ptmgr->_delEntry( TEST_DATA[x].va ); + + uint64_t status = ptmgr->_getStatus( TEST_DATA[x].va, ignored ); + if( status & PageTableManager::PTE_VALID ) + { + TS_FAIL( "ptmgrtest::test_delEntry> ERROR : entry still present!" ); + printk( "VA = %.16lX\n", TEST_DATA[x].va ); + PageTableManager::printPTE( TEST_DATA[x].va, true ); + fails++; + } + else + { + PASS_TRACE( "ptmgrtest::test_delEntry> PASS : 0x%.16lX", TEST_DATA[x].va ); + } + total++; + } + + // should print out an empty table + //ptmgr->_printPT(); + + delete ptmgr; + + TS_TRACE( "ptmgrtest::test_delEntry> fails=%d/%d", fails, total ); + printk( "ptmgrtest::test_delEntry> fails=%ld/%ld\n", fails, total ); + } + + /** + * 1) Initialize the Page Table + * 2) Populate the Page Table + * 3) Remove a range of PTEs + * 4) Verify they have been removed + */ + void test_delRangeVA( void ) + { + TS_TRACE( ">> test_delRangeVA <<" ); + printk( ">> test_delRangeVA <<\n" ); + uint64_t fails = 0; + uint64_t total = 0; + uint64_t ignored = 0; + + // 1) Initialize the Page Table + PageTableManager* ptmgr = new PageTableManager(true); + + // 2) Populate the Page Table + fillTable(ptmgr); + //ptmgr->_printPT(); + + // 3) Remove a range of PTEs + ptmgr->_delRangeVA( VA_RANGE_START, VA_RANGE_FINISH ); + + // 4) Verify they have been removed + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + if( TEST_DATA[x].loaded ) + { + uint64_t status = ptmgr->_getStatus( TEST_DATA[x].va, ignored ); + if( (status & PageTableManager::PTE_VALID) && TEST_DATA[x].delrangeva ) + { + TS_FAIL( "ptmgrtest::test_delRangeVA> ERROR1 : entry still present" ); + printk( "addr = %.16lX\n", TEST_DATA[x].va ); + PageTableManager::printPTE( TEST_DATA[x].va, false ); + fails++; + } + else if( !(status & PageTableManager::PTE_VALID) && !TEST_DATA[x].delrangeva ) + { + TS_FAIL( "ptmgrtest::test_delRangeVA> ERROR2 : deleted a wrong entry" ); + printk( "addr = %.16lX\n", TEST_DATA[x].va ); + PageTableManager::printPTE( TEST_DATA[x].va, false ); + fails++; + } + else + { + PASS_TRACE( "%s> PASS : 0x%.16lX\n", __FUNCTION__, TEST_DATA[x].va ); + } + total++; + } + } + + //ptmgr->_printPT(); + + delete ptmgr; + + TS_TRACE( "ptmgrtest::test_delRangeVA> fails=%d/%d", fails, total ); + printk( "ptmgrtest::test_delRangeVA> fails=%ld/%ld\n", fails, total ); + } + + /** + * 1) Initialize the Page Table + * 2) Populate the Page Table + * 3) Remove a range of PTEs + * 4) Verify they have been removed + */ + void test_delRangePN( void ) + { + TS_TRACE( ">> test_delRangePN <<" ); + printk( ">> test_delRangePN <<\n" ); + uint64_t fails = 0; + uint64_t total = 0; + uint64_t pagenum = 0; + + // 1) Initialize the Page Table + PageTableManager* ptmgr = new PageTableManager(true); + + // 2) Populate the Page Table + fillTable(ptmgr); + //ptmgr->_printPT(); + + // 3) Remove a range of PTEs + ptmgr->_delRangePN( PN_RANGE_START, PN_RANGE_FINISH ); + + // 4) Verify they have been removed + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + if( TEST_DATA[x].loaded ) + { + uint64_t status = ptmgr->_getStatus( TEST_DATA[x].va, pagenum ); + if( (status & PageTableManager::PTE_VALID) + && (pagenum >= PN_RANGE_START) + && (pagenum <= PN_RANGE_FINISH) ) + { + TS_FAIL( "ptmgrtest::test_delRangePN> ERROR1 : entry still present" ); + printk( "addr = %.16lX, page=%ld\n", TEST_DATA[x].va, pagenum ); + PageTableManager::printPTE( TEST_DATA[x].va, false ); + fails++; + } + else if( !(status & PageTableManager::PTE_VALID) + && (pagenum < PN_RANGE_START) + && (pagenum > PN_RANGE_FINISH) ) + { + TS_FAIL( "ptmgrtest::test_delRangePN> ERROR2 : deleted a wrong entry" ); + printk( "addr = %.16lX, page=%ld\n", TEST_DATA[x].va, pagenum ); + PageTableManager::printPTE( TEST_DATA[x].va, false ); + fails++; + } + else + { + PASS_TRACE( "%s> PASS : 0x%.16lX\n", __FUNCTION__, TEST_DATA[x].va ); + } + total++; + } + } + + //ptmgr->_printPT(); + + delete ptmgr; + + TS_TRACE( "ptmgrtest::test_delRangePN> fails=%d/%d", fails, total ); + printk( "ptmgrtest::test_delRangePN> fails=%ld/%ld\n", fails, total ); + } + + //@todo - Test LRU + + private: + /** + * @brief Populate the table with entries + */ + void fillTable( PageTableManager* ptmgr ) + { + for( uint64_t x = 0; x < (sizeof(TEST_DATA)/sizeof(TEST_DATA[0])); x++ ) + { + // 2) Add some PTEs + ptmgr->_addEntry( TEST_DATA[x].va, TEST_DATA[x].page, VmmManager::NORMAL_ACCESS ); + } + } + +}; + + +#endif -- cgit v1.2.1