diff options
| -rw-r--r-- | arch/mips/Kconfig | 13 | ||||
| -rw-r--r-- | arch/mips/include/asm/pgtable-64.h | 2 | ||||
| -rw-r--r-- | arch/mips/include/asm/pgtable-bits.h | 11 | ||||
| -rw-r--r-- | arch/mips/include/asm/pgtable.h | 167 | ||||
| -rw-r--r-- | arch/mips/kernel/mips_ksyms.c | 2 | ||||
| -rw-r--r-- | arch/mips/mm/pgtable-64.c | 31 | ||||
| -rw-r--r-- | arch/mips/mm/tlb-r4k.c | 20 | ||||
| -rw-r--r-- | arch/mips/mm/tlbex.c | 3 |
8 files changed, 243 insertions, 6 deletions
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index b47d591c03dd..9934a4687ac4 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -19,6 +19,7 @@ config MIPS select HAVE_KRETPROBES select HAVE_DEBUG_KMEMLEAK select ARCH_BINFMT_ELF_RANDOMIZE_PIE + select HAVE_ARCH_TRANSPARENT_HUGEPAGE select RTC_LIB if !MACH_LOONGSON select GENERIC_ATOMIC64 if !64BIT select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE @@ -1372,6 +1373,7 @@ config CPU_R4X00 depends on SYS_HAS_CPU_R4X00 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HUGEPAGES help MIPS Technologies R4000-series processors other than 4300, including the R4000, R4400, R4600, and 4700. @@ -1382,12 +1384,14 @@ config CPU_TX49XX select CPU_HAS_PREFETCH select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HUGEPAGES config CPU_R5000 bool "R5000" depends on SYS_HAS_CPU_R5000 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HUGEPAGES help MIPS Technologies R5000-series processors other than the Nevada. @@ -1396,6 +1400,7 @@ config CPU_R5432 depends on SYS_HAS_CPU_R5432 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HUGEPAGES config CPU_R5500 bool "R5500" @@ -1421,6 +1426,7 @@ config CPU_NEVADA depends on SYS_HAS_CPU_NEVADA select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HUGEPAGES help QED / PMC-Sierra RM52xx-series ("Nevada") processors. @@ -1441,6 +1447,7 @@ config CPU_R10000 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES help MIPS Technologies R10000-series processors. @@ -1451,6 +1458,7 @@ config CPU_RM7000 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES config CPU_RM9000 bool "RM9000" @@ -1459,6 +1467,7 @@ config CPU_RM9000 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES select WEAK_ORDERING config CPU_SB1 @@ -1467,6 +1476,7 @@ config CPU_SB1 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES select WEAK_ORDERING config CPU_CAVIUM_OCTEON @@ -1530,9 +1540,9 @@ config CPU_XLR select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES select WEAK_ORDERING select WEAK_REORDERING_BEYOND_LLSC - select CPU_SUPPORTS_HUGEPAGES help Netlogic Microsystems XLR/XLS processors. @@ -1593,6 +1603,7 @@ config CPU_LOONGSON2 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES config CPU_LOONGSON1 bool diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index f5b521d5a67d..c63191055e69 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -175,7 +175,7 @@ static inline int pmd_none(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { -#ifdef CONFIG_HUGETLB_PAGE +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT /* pmd_huge(pmd) but inline */ if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) return 0; diff --git a/arch/mips/include/asm/pgtable-bits.h b/arch/mips/include/asm/pgtable-bits.h index 1e2642c360a8..9ce1ac782448 100644 --- a/arch/mips/include/asm/pgtable-bits.h +++ b/arch/mips/include/asm/pgtable-bits.h @@ -137,8 +137,17 @@ #define _PAGE_HUGE ({BUG(); 1; }) /* Dummy value */ #endif +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT +/* huge tlb page */ +#define _PAGE_SPLITTING_SHIFT (_PAGE_HUGE_SHIFT + 1) +#define _PAGE_SPLITTING (1 << _PAGE_SPLITTING_SHIFT) +#else +#define _PAGE_SPLITTING_SHIFT (_PAGE_HUGE_SHIFT) +#define _PAGE_SPLITTING ({BUG(); 1; }) /* Dummy value */ +#endif + /* Page cannot be executed */ -#define _PAGE_NO_EXEC_SHIFT (cpu_has_rixi ? _PAGE_HUGE_SHIFT + 1 : _PAGE_HUGE_SHIFT) +#define _PAGE_NO_EXEC_SHIFT (cpu_has_rixi ? _PAGE_SPLITTING_SHIFT + 1 : _PAGE_SPLITTING_SHIFT) #define _PAGE_NO_EXEC ({BUG_ON(!cpu_has_rixi); 1 << _PAGE_NO_EXEC_SHIFT; }) /* Page cannot be read */ diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index bc1c0fbf8d99..252202d24a84 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -8,6 +8,7 @@ #ifndef _ASM_PGTABLE_H #define _ASM_PGTABLE_H +#include <linux/mmzone.h> #ifdef CONFIG_32BIT #include <asm/pgtable-32.h> #endif @@ -94,7 +95,12 @@ extern void paging_init(void); * and a page entry and page directory to the page they refer to. */ #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) -#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) + +#define __pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) +#ifndef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_page(pmd) __pmd_page(pmd) +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + #define pmd_page_vaddr(pmd) pmd_val(pmd) #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) @@ -374,6 +380,14 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, __update_cache(vma, address, pte); } +static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + pte_t pte = *(pte_t *)pmdp; + + __update_tlb(vma, address, pte); +} + #define kern_addr_valid(addr) (1) #ifdef CONFIG_64BIT_PHYS_ADDR @@ -393,6 +407,157 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, remap_pfn_range(vma, vaddr, pfn, size, prot) #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +extern int has_transparent_hugepage(void); + +static inline int pmd_trans_huge(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_HUGE); +} + +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_HUGE; + + return pmd; +} + +static inline int pmd_trans_splitting(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_SPLITTING); +} + +static inline pmd_t pmd_mksplitting(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_SPLITTING; + + return pmd; +} + +extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); + +#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH +/* Extern to avoid header file madness */ +extern void pmdp_splitting_flush(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp); + +#define __HAVE_ARCH_PMD_WRITE +static inline int pmd_write(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_WRITE); +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_SILENT_WRITE); + return pmd; +} + +static inline pmd_t pmd_mkwrite(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_WRITE; + if (pmd_val(pmd) & _PAGE_MODIFIED) + pmd_val(pmd) |= _PAGE_SILENT_WRITE; + + return pmd; +} + +static inline int pmd_dirty(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_MODIFIED); +} + +static inline pmd_t pmd_mkclean(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_MODIFIED | _PAGE_SILENT_WRITE); + return pmd; +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_MODIFIED; + if (pmd_val(pmd) & _PAGE_WRITE) + pmd_val(pmd) |= _PAGE_SILENT_WRITE; + + return pmd; +} + +static inline int pmd_young(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_ACCESSED); +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_ACCESSED|_PAGE_SILENT_READ); + + return pmd; +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_ACCESSED; + + if (cpu_has_rixi) { + if (!(pmd_val(pmd) & _PAGE_NO_READ)) + pmd_val(pmd) |= _PAGE_SILENT_READ; + } else { + if (pmd_val(pmd) & _PAGE_READ) + pmd_val(pmd) |= _PAGE_SILENT_READ; + } + + return pmd; +} + +/* Extern to avoid header file madness */ +extern pmd_t mk_pmd(struct page *page, pgprot_t prot); + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return pmd_val(pmd) >> _PFN_SHIFT; +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + if (pmd_trans_huge(pmd)) + return pfn_to_page(pmd_pfn(pmd)); + + return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot); + return pmd; +} + +static inline pmd_t pmd_mknotpresent(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY); + + return pmd; +} + +/* + * The generic version pmdp_get_and_clear uses a version of pmd_clear() with a + * different prototype. + */ +#define __HAVE_ARCH_PMDP_GET_AND_CLEAR +static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + pmd_t old = *pmdp; + + pmd_clear(pmdp); + + return old; +} + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + #include <asm-generic/pgtable.h> /* diff --git a/arch/mips/kernel/mips_ksyms.c b/arch/mips/kernel/mips_ksyms.c index 3fc1691110dc..1ba8933683aa 100644 --- a/arch/mips/kernel/mips_ksyms.c +++ b/arch/mips/kernel/mips_ksyms.c @@ -11,7 +11,7 @@ #include <linux/interrupt.h> #include <linux/export.h> #include <asm/checksum.h> -#include <asm/pgtable.h> +#include <linux/mm.h> #include <asm/uaccess.h> #include <asm/ftrace.h> diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c index 25407794edb4..ee331bbd8f8a 100644 --- a/arch/mips/mm/pgtable-64.c +++ b/arch/mips/mm/pgtable-64.c @@ -11,6 +11,7 @@ #include <asm/fixmap.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> +#include <asm/tlbflush.h> void pgd_init(unsigned long page) { @@ -61,6 +62,36 @@ void pmd_init(unsigned long addr, unsigned long pagetable) } #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +void pmdp_splitting_flush(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + if (!pmd_trans_splitting(*pmdp)) { + pmd_t pmd = pmd_mksplitting(*pmdp); + set_pmd_at(vma->vm_mm, address, pmdp, pmd); + } +} + +#endif + +pmd_t mk_pmd(struct page *page, pgprot_t prot) +{ + pmd_t pmd; + + pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); + + return pmd; +} + +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; + flush_tlb_all(); +} + void __init pagetable_init(void) { unsigned long vaddr; diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 936165d167e1..94ad86d055c5 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -377,6 +377,26 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, EXIT_CRITICAL(flags); } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +int __init has_transparent_hugepage(void) +{ + unsigned int mask; + unsigned long flags; + + ENTER_CRITICAL(flags); + write_c0_pagemask(PM_HUGE_MASK); + back_to_back_c0_hazard(); + mask = read_c0_pagemask(); + write_c0_pagemask(PM_DEFAULT_MASK); + + EXIT_CRITICAL(flags); + + return mask == PM_HUGE_MASK; +} + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + static int __cpuinitdata ntlb; static int __init set_ntlb(char *str) { diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 98b2b732005a..6af62a2fec21 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -225,8 +225,9 @@ static void output_pgtable_bits_defines(void) pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); -#ifdef _PAGE_HUGE_SHIFT +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); + pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT); #endif if (cpu_has_rixi) { #ifdef _PAGE_NO_EXEC_SHIFT |
