diff options
| -rw-r--r-- | arch/ia64/mm/hugetlbpage.c | 29 | ||||
| -rw-r--r-- | arch/ppc64/mm/hugetlbpage.c | 10 | ||||
| -rw-r--r-- | include/asm-ia64/page.h | 2 | ||||
| -rw-r--r-- | include/asm-ia64/pgtable.h | 4 | ||||
| -rw-r--r-- | include/asm-ppc64/pgtable.h | 12 | ||||
| -rw-r--r-- | include/linux/hugetlb.h | 6 | ||||
| -rw-r--r-- | include/linux/mm.h | 4 | ||||
| -rw-r--r-- | mm/memory.c | 36 |
8 files changed, 65 insertions, 38 deletions
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c index 626258ae9742..df08ae7634b6 100644 --- a/arch/ia64/mm/hugetlbpage.c +++ b/arch/ia64/mm/hugetlbpage.c @@ -186,13 +186,30 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int wri return NULL; } -/* - * Do nothing, until we've worked out what to do! To allow build, we - * must remove reference to clear_page_range since it no longer exists. - */ -void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev, - unsigned long start, unsigned long end) +void hugetlb_free_pgd_range(struct mmu_gather **tlb, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) { + /* + * This is called only when is_hugepage_only_range(addr,), + * and it follows that is_hugepage_only_range(end,) also. + * + * The offset of these addresses from the base of the hugetlb + * region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that + * the standard free_pgd_range will free the right page tables. + * + * If floor and ceiling are also in the hugetlb region, they + * must likewise be scaled down; but if outside, left unchanged. + */ + + addr = htlbpage_to_page(addr); + end = htlbpage_to_page(end); + if (is_hugepage_only_range(tlb->mm, floor, HPAGE_SIZE)) + floor = htlbpage_to_page(floor); + if (is_hugepage_only_range(tlb->mm, ceiling, HPAGE_SIZE)) + ceiling = htlbpage_to_page(ceiling); + + free_pgd_range(tlb, addr, end, floor, ceiling); } void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c index c62ddaff0720..8665bb57e42b 100644 --- a/arch/ppc64/mm/hugetlbpage.c +++ b/arch/ppc64/mm/hugetlbpage.c @@ -430,16 +430,6 @@ void unmap_hugepage_range(struct vm_area_struct *vma, flush_tlb_pending(); } -void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev, - unsigned long start, unsigned long end) -{ - /* Because the huge pgtables are only 2 level, they can take - * at most around 4M, much less than one hugepage which the - * process is presumably entitled to use. So we don't bother - * freeing up the pagetables on unmap, and wait until - * destroy_context() to clean up the lot. */ -} - int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma) { struct mm_struct *mm = current->mm; diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h index 24aab801a8ca..08894f73abf0 100644 --- a/include/asm-ia64/page.h +++ b/include/asm-ia64/page.h @@ -139,7 +139,7 @@ typedef union ia64_va { # define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) # define is_hugepage_only_range(mm, addr, len) \ (REGION_NUMBER(addr) == REGION_HPAGE && \ - REGION_NUMBER((addr)+(len)) == REGION_HPAGE) + REGION_NUMBER((addr)+(len)-1) == REGION_HPAGE) extern unsigned int hpage_shift; #endif diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h index 1757a811f436..bbf6dd757003 100644 --- a/include/asm-ia64/pgtable.h +++ b/include/asm-ia64/pgtable.h @@ -472,8 +472,8 @@ extern struct page *zero_page_memmap_ptr; #define HUGETLB_PGDIR_SIZE (__IA64_UL(1) << HUGETLB_PGDIR_SHIFT) #define HUGETLB_PGDIR_MASK (~(HUGETLB_PGDIR_SIZE-1)) struct mmu_gather; -extern void hugetlb_free_pgtables(struct mmu_gather *tlb, - struct vm_area_struct * prev, unsigned long start, unsigned long end); +void hugetlb_free_pgd_range(struct mmu_gather **tlb, unsigned long addr, + unsigned long end, unsigned long floor, unsigned long ceiling); #endif /* diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h index 4c4824653e80..33b90e2aa47d 100644 --- a/include/asm-ppc64/pgtable.h +++ b/include/asm-ppc64/pgtable.h @@ -500,9 +500,15 @@ extern pgd_t ioremap_dir[1024]; extern void paging_init(void); -struct mmu_gather; -void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev, - unsigned long start, unsigned long end); +/* + * Because the huge pgtables are only 2 level, they can take + * at most around 4M, much less than one hugepage which the + * process is presumably entitled to use. So we don't bother + * freeing up the pagetables on unmap, and wait until + * destroy_context() to clean up the lot. + */ +#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \ + do { } while (0) /* * This gets called at the end of handling a page fault, when diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ae45676d27ba..6af1ae4a8211 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -37,7 +37,8 @@ extern int sysctl_hugetlb_shm_group; #ifndef ARCH_HAS_HUGEPAGE_ONLY_RANGE #define is_hugepage_only_range(mm, addr, len) 0 -#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0) +#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \ + do { } while (0) #endif #ifndef ARCH_HAS_PREPARE_HUGEPAGE_RANGE @@ -72,7 +73,8 @@ static inline unsigned long hugetlb_total_pages(void) #define prepare_hugepage_range(addr, len) (-EINVAL) #define pmd_huge(x) 0 #define is_hugepage_only_range(mm, addr, len) 0 -#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0) +#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \ + do { } while (0) #define alloc_huge_page() ({ NULL; }) #define free_huge_page(p) ({ (void)(p); BUG(); }) diff --git a/include/linux/mm.h b/include/linux/mm.h index 59eca28b5ae2..c74a74ca401d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -587,7 +587,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlb, struct mm_struct *mm, struct vm_area_struct *start_vma, unsigned long start_addr, unsigned long end_addr, unsigned long *nr_accounted, struct zap_details *); -void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma, +void free_pgd_range(struct mmu_gather **tlb, unsigned long addr, + unsigned long end, unsigned long floor, unsigned long ceiling); +void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *start_vma, unsigned long floor, unsigned long ceiling); int copy_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma); diff --git a/mm/memory.c b/mm/memory.c index 854bd90eeca1..6bad4c4064e7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -190,7 +190,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, * * Must be called with pagetable lock held. */ -static inline void free_pgd_range(struct mmu_gather *tlb, +void free_pgd_range(struct mmu_gather **tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling) { @@ -241,37 +241,47 @@ static inline void free_pgd_range(struct mmu_gather *tlb, return; start = addr; - pgd = pgd_offset(tlb->mm, addr); + pgd = pgd_offset((*tlb)->mm, addr); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; - free_pud_range(tlb, pgd, addr, next, floor, ceiling); + free_pud_range(*tlb, pgd, addr, next, floor, ceiling); } while (pgd++, addr = next, addr != end); - if (!tlb_is_full_mm(tlb)) - flush_tlb_pgtables(tlb->mm, start, end); + if (!tlb_is_full_mm(*tlb)) + flush_tlb_pgtables((*tlb)->mm, start, end); } void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma, - unsigned long floor, unsigned long ceiling) + unsigned long floor, unsigned long ceiling) { while (vma) { struct vm_area_struct *next = vma->vm_next; unsigned long addr = vma->vm_start; - /* Optimization: gather nearby vmas into a single call down */ - while (next && next->vm_start <= vma->vm_end + PMD_SIZE) { - vma = next; - next = vma->vm_next; - } - free_pgd_range(*tlb, addr, vma->vm_end, + if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) { + hugetlb_free_pgd_range(tlb, addr, vma->vm_end, + floor, next? next->vm_start: ceiling); + } else { + /* + * Optimization: gather nearby vmas into one call down + */ + while (next && next->vm_start <= vma->vm_end + PMD_SIZE + && !is_hugepage_only_range(vma->vm_mm, next->vm_start, + HPAGE_SIZE)) { + vma = next; + next = vma->vm_next; + } + free_pgd_range(tlb, addr, vma->vm_end, floor, next? next->vm_start: ceiling); + } vma = next; } } -pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address) +pte_t fastcall *pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, + unsigned long address) { if (!pmd_present(*pmd)) { struct page *new; |
